In [1]:
import sys
sys.path.insert(0, '../')
import pandas as pd
import numpy as np
np.set_printoptions(precision=3, linewidth=200, suppress=True)
from library.datasets.cifar10 import CIFAR10
from library.plot_tools import plot_tools
from sklearn.model_selection import KFold, train_test_split, GridSearchCV
from sklearn.model_selection import cross_val_score, learning_curve
from sklearn.metrics import confusion_matrix, classification_report
from sklearn.externals import joblib
import time
from sklearn.preprocessing import StandardScaler, MinMaxScaler
import sklearn.metrics as skm
import matplotlib.pyplot as plt
from library.utils import file_utils
from scipy.misc import toimage
%matplotlib inline
None
In [2]:
from sklearn.neural_network import MLPClassifier
In [3]:
total_time = 0
exp_no = 103
file_no = 7
output_directory = '../logs/cifar10/' + str(file_no).zfill(2) + '_mlp_raw_cross_val/' + 'exp_no_' + str(exp_no).zfill(3) + '/'
In [4]:
data_source = 'Website'
search_method = 'grid'
train_validate_split_data=None
one_hot=False
In [5]:
total_time = 0
mlp_max_iter = 1000
exp_jobs = 10
num_images_required = 0.2
num_folds = 10
In [6]:
param_grid = [
{'hidden_layer_sizes': [(3072,3072), (4000, 4000)],
'solver': ['lbfgs', 'adam'],
'alpha': [1e-5, 1e-6, 1e-7]
},
]
param_name = 'exp_' + str(exp_no).zfill(3)
In [7]:
start = time.time()
cifar10 = CIFAR10(one_hot_encode=one_hot, num_images=num_images_required,
train_validate_split=train_validate_split_data, endian='little')
cifar10.load_data(train=True, test=True, data_directory='./datasets/cifar10/')
end = time.time()
print('[ Step 0] Dataset loaded in %5.6f ms' %((end-start)*1000))
print('Dataset size: ' + str(cifar10.train.data.shape))
num_train_images = cifar10.train.data.shape[0]
total_time += (end-start)
Loading CIFAR 10 Dataset
Downloading and extracting CIFAR 10 file
MD5sum of the file: ./datasets/cifar10/cifar-10.tar.gz is verified
Loading 10000 train images
Loading CIFAR 10 Training Dataset
Reading unpicked data file: ./datasets/cifar10/cifar-10-batches/data_batch_1
Reading unpicked data file: ./datasets/cifar10/cifar-10-batches/data_batch_2
Reading unpicked data file: ./datasets/cifar10/cifar-10-batches/data_batch_3
Reading unpicked data file: ./datasets/cifar10/cifar-10-batches/data_batch_4
Reading unpicked data file: ./datasets/cifar10/cifar-10-batches/data_batch_5
Loading 10000 test images
Loading CIFAR 10 Test Dataset
Unpickling test file: ./datasets/cifar10/cifar-10-batches/test_batch
Reading unpicked test file: ./datasets/cifar10/cifar-10-batches/test_batch
Loaded CIFAR 10 Dataset in 2.1444 seconds
[ Step 0] Dataset loaded in 2145.123720 ms
Dataset size: (10000, 3072)
In [8]:
cifar10.plot_sample(plot_data=True, plot_test=True, fig_size=(7, 7))
Plotting CIFAR 10 Train Dataset
Plotting CIFAR 10 Test Dataset
In [9]:
cifar10.plot_images(cifar10.train.data[:50, :], cifar10.train.class_names[:50],
nrows=5, ncols=10, fig_size=(20,50), fontsize=35, convert=True)
Out[9]:
True
In [10]:
print('Training images')
print(cifar10.train.data[:5])
if one_hot is True:
print('Training labels')
print(cifar10.train.one_hot_labels[:5])
print('Training classes')
print(cifar10.train.class_labels[:5])
print('Testing images')
print(cifar10.test.data[:5])
if one_hot is True:
print('Testing labels')
print(cifar10.test.one_hot_labels[:5])
print('Testing classes')
print(cifar10.test.class_labels[:5])
print('[ Step 0.1] Working with only %d images' %num_train_images)
Training images
[[ 59 43 50 ..., 140 84 72]
[154 126 105 ..., 139 142 144]
[255 253 253 ..., 83 83 84]
[ 28 37 38 ..., 28 37 46]
[170 168 177 ..., 82 78 80]]
Training classes
[6 9 9 4 1]
Testing images
[[158 159 165 ..., 124 129 110]
[235 231 232 ..., 178 191 199]
[158 158 139 ..., 8 3 7]
[155 167 176 ..., 50 52 50]
[ 65 70 48 ..., 136 146 117]]
Testing classes
[3 8 8 0 6]
[ Step 0.1] Working with only 10000 images
In [11]:
start = time.time()
ss = StandardScaler()
data_images = ss.fit_transform(cifar10.train.data)
test_images = ss.fit_transform(cifar10.test.data)
end = time.time()
print('[ Step 1] Dataset transformations done in %5.6f ms' %((end-start)*1000))
print('Training the classifier on %d images' % num_train_images)
print('Dataset size: ' + str(cifar10.train.data.shape))
total_time += (end-start)
/net/voxel03/misc/me/praneethas/Softwares/anaconda3/lib/python3.6/site-packages/sklearn/utils/validation.py:429: DataConversionWarning: Data with input dtype uint8 was converted to float64 by StandardScaler.
warnings.warn(msg, _DataConversionWarning)
[ Step 1] Dataset transformations done in 816.001415 ms
Training the classifier on 10000 images
Dataset size: (10000, 3072)
In [12]:
print('Parameters to serach for')
print('\n'.join([str(param) for param in param_grid])); print()
Parameters to serach for
{'hidden_layer_sizes': [(3072, 3072), (4000, 4000)], 'solver': ['lbfgs', 'adam'], 'alpha': [1e-05, 1e-06, 1e-07]}
In [13]:
scores = []
scores_mean = []
scores_std = []
In [ ]:
mlp_clf = MLPClassifier(random_state=0, max_iter=mlp_max_iter, verbose=True)
print(mlp_clf)
MLPClassifier(activation='relu', alpha=0.0001, batch_size='auto', beta_1=0.9,
beta_2=0.999, early_stopping=False, epsilon=1e-08,
hidden_layer_sizes=(100,), learning_rate='constant',
learning_rate_init=0.001, max_iter=1000, momentum=0.9,
nesterovs_momentum=True, power_t=0.5, random_state=0, shuffle=True,
solver='adam', tol=0.0001, validation_fraction=0.1, verbose=True,
warm_start=False)
In [ ]:
start = time.time()
if search_method == 'grid':
print('Applying GridSearchCV')
estimator = GridSearchCV(mlp_clf, param_grid, cv=num_folds, scoring='accuracy', verbose=3, n_jobs=exp_jobs)
elif search_method == 'random':
print('Applying RandomizedSearchCV')
estimator = RandomizedSearchCV(mlp_clf, param_grid, cv=num_folds, scoring='accuracy', n_iter=10,
random_state=0, verbose=3, n_jobs=exp_jobs)
else:
print('Applying GridSearchCV')
estimator = GridSearchCV(mlp_clf, param_grid, cv=num_folds, scoring='accuracy', verbose=3, n_jobs=exp_jobs)
print(estimator)
estimator_result = estimator.fit(data_images, cifar10.train.class_labels)
end = time.time()
total_time += (end-start)
print('Total Time taken for cross validation and finding best parameters: %.4f ms' %((end-start)*1000))
Applying GridSearchCV
GridSearchCV(cv=10, error_score='raise',
estimator=MLPClassifier(activation='relu', alpha=0.0001, batch_size='auto', beta_1=0.9,
beta_2=0.999, early_stopping=False, epsilon=1e-08,
hidden_layer_sizes=(100,), learning_rate='constant',
learning_rate_init=0.001, max_iter=1000, momentum=0.9,
nesterovs_momentum=True, power_t=0.5, random_state=0, shuffle=True,
solver='adam', tol=0.0001, validation_fraction=0.1, verbose=True,
warm_start=False),
fit_params={}, iid=True, n_jobs=10,
param_grid=[{'hidden_layer_sizes': [(3072, 3072), (4000, 4000)], 'solver': ['lbfgs', 'adam'], 'alpha': [1e-05, 1e-06, 1e-07]}],
pre_dispatch='2*n_jobs', refit=True, return_train_score=True,
scoring='accuracy', verbose=3)
Fitting 10 folds for each of 12 candidates, totalling 120 fits
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=adam .......
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=adam .......
Iteration 1, loss = 2.95767506
Iteration 1, loss = 2.88678346
Iteration 2, loss = 1.66452353
Iteration 2, loss = 1.68803758
Iteration 3, loss = 1.52012973
Iteration 3, loss = 1.50214320
Iteration 4, loss = 1.37886495
Iteration 4, loss = 1.38252807
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=adam .......
Iteration 5, loss = 1.26446154
Iteration 5, loss = 1.24476744
Iteration 6, loss = 1.13382146
Iteration 6, loss = 1.14528918
Iteration 1, loss = 2.97348178
Iteration 7, loss = 1.03115601
Iteration 7, loss = 1.05542932
Iteration 2, loss = 1.64944376
Iteration 8, loss = 0.91956540
Iteration 8, loss = 0.92883509
Iteration 3, loss = 1.49118392
Iteration 9, loss = 0.82198534
Iteration 9, loss = 0.78000794
Iteration 4, loss = 1.34787990
Iteration 10, loss = 0.69327692
Iteration 10, loss = 0.69168605
Iteration 5, loss = 1.22000951
Iteration 11, loss = 0.61258640
Iteration 11, loss = 0.61084990
Iteration 6, loss = 1.10637566
Iteration 12, loss = 0.55806115
Iteration 12, loss = 0.50927772
Iteration 7, loss = 1.01183225
Iteration 13, loss = 0.44137696
Iteration 13, loss = 0.48443015
Iteration 8, loss = 0.86185308
Iteration 14, loss = 0.45188308
Iteration 14, loss = 0.38122859
Iteration 9, loss = 0.75689083
Iteration 15, loss = 0.34761229
Iteration 15, loss = 0.34460100
Iteration 10, loss = 0.63876703
Iteration 16, loss = 0.29993314
Iteration 16, loss = 0.32744354
Iteration 11, loss = 0.59105375
Iteration 17, loss = 0.25331432
Iteration 17, loss = 0.26900113
Iteration 12, loss = 0.48792301
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=adam .......
Iteration 18, loss = 0.24739446
Iteration 18, loss = 0.22427166
Iteration 13, loss = 0.46726954
Iteration 1, loss = 2.99787782
Iteration 19, loss = 0.28927826
Iteration 19, loss = 0.20272217
Iteration 14, loss = 0.35245308
Iteration 2, loss = 1.69027154
Iteration 20, loss = 0.18589363
Iteration 20, loss = 0.30820780
Iteration 15, loss = 0.30292829
Iteration 3, loss = 1.53678105
Iteration 21, loss = 0.23933570
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
Iteration 21, loss = 0.19117419
Iteration 16, loss = 0.28347393
Iteration 4, loss = 1.41629810
Iteration 22, loss = 0.17948295
Iteration 17, loss = 0.26051803
Iteration 5, loss = 1.27366233
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.470588, total=62.1min
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=adam .......
Iteration 23, loss = 0.20328700
Iteration 18, loss = 0.28253538
Iteration 6, loss = 1.17028741
Iteration 1, loss = 2.75452728
Iteration 24, loss = 0.22310448
Iteration 19, loss = 0.28433589
Iteration 7, loss = 1.02796287
Iteration 2, loss = 1.64208625
Iteration 25, loss = 0.18018920
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.427861, total=78.0min
Iteration 20, loss = 0.22360253
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=adam .......
Iteration 8, loss = 0.90902537
Iteration 3, loss = 1.48509154
Iteration 21, loss = 0.17664800
Iteration 1, loss = 2.96938292
Iteration 9, loss = 0.80580612
Iteration 4, loss = 1.34520473
Iteration 22, loss = 0.16463034
Iteration 2, loss = 1.72672107
Iteration 10, loss = 0.72953871
Iteration 5, loss = 1.21277824
Iteration 23, loss = 0.13856350
Iteration 3, loss = 1.65684847
Iteration 11, loss = 0.63018022
Iteration 6, loss = 1.08473291
Iteration 24, loss = 0.15781340
Iteration 4, loss = 1.59178554
Iteration 12, loss = 0.54244409
Iteration 7, loss = 0.97231094
Iteration 25, loss = 0.11937858
Iteration 5, loss = 1.43353110
Iteration 13, loss = 0.46211374
Iteration 8, loss = 0.86880507
Iteration 26, loss = 0.16140365
Iteration 6, loss = 1.40616861
Iteration 14, loss = 0.41150157
Iteration 9, loss = 0.73545943
Iteration 27, loss = 0.16947168
Iteration 7, loss = 1.20235287
Iteration 15, loss = 0.30450110
Iteration 10, loss = 0.66881313
Iteration 28, loss = 0.15528256
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.442116, total=55.5min
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=adam .......
Iteration 8, loss = 1.09276746
Iteration 16, loss = 0.26635457
Iteration 11, loss = 0.58068801
Iteration 1, loss = 2.81133842
Iteration 9, loss = 1.00533843
Iteration 17, loss = 0.26414710
Iteration 12, loss = 0.51354768
Iteration 2, loss = 1.75138060
Iteration 10, loss = 1.15811994
Iteration 18, loss = 0.25589655
Iteration 13, loss = 0.44497566
Iteration 3, loss = 1.63866287
Iteration 11, loss = 0.87766122
Iteration 19, loss = 0.23518134
Iteration 14, loss = 0.34890708
Iteration 4, loss = 1.49590908
Iteration 12, loss = 1.18169837
Iteration 20, loss = 0.20923809
Iteration 15, loss = 0.34107969
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=lbfgs, score=0.429288, total=176.2min
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=adam .......
Iteration 5, loss = 1.35587917
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=lbfgs, score=0.442116, total=177.2min
Iteration 13, loss = 0.80195741
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=adam .......
Iteration 21, loss = 0.18138777
Iteration 16, loss = 0.25936738
Iteration 1, loss = 3.06373025
Iteration 6, loss = 1.28598825
Iteration 14, loss = 0.91610987
Iteration 22, loss = 0.21098493
Iteration 1, loss = 2.86396357
Iteration 17, loss = 0.23307580
Iteration 2, loss = 1.71054133
Iteration 7, loss = 1.27582962
Iteration 15, loss = 1.22988034
Iteration 23, loss = 0.19629766
Iteration 2, loss = 1.70761924
Iteration 18, loss = 0.23665116
Iteration 3, loss = 1.64554574
Iteration 8, loss = 1.48760076
Iteration 16, loss = 0.85416784
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
Iteration 3, loss = 1.57999044
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.416416, total=26.2min
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=adam .......
Iteration 19, loss = 0.23553498
Iteration 24, loss = 0.22143599
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.438124, total=34.7min
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
Iteration 4, loss = 1.50142580
Iteration 9, loss = 1.22594722
Iteration 4, loss = 1.45041598
Iteration 1, loss = 2.83455023
Iteration 20, loss = 0.22507838
Iteration 5, loss = 1.43160043
Iteration 10, loss = 1.06316630
Iteration 5, loss = 1.43491198
Iteration 2, loss = 1.73097911
Iteration 21, loss = 0.21681501
Iteration 6, loss = 1.31140437
Iteration 11, loss = 1.13487151
Iteration 3, loss = 1.58165863
Iteration 6, loss = 1.29511776
Iteration 22, loss = 0.18441552
Iteration 7, loss = 1.22279537
Iteration 12, loss = 1.16669151
Iteration 4, loss = 1.46308857
Iteration 7, loss = 1.21205917
Iteration 23, loss = 0.17375250
Iteration 8, loss = 1.18581863
Iteration 13, loss = 1.20807641
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
Iteration 5, loss = 1.34505702
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.450902, total=34.3min
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
Iteration 8, loss = 1.09296293
Iteration 24, loss = 0.14419402
Iteration 9, loss = 1.22215959
Iteration 6, loss = 1.20969489
Iteration 9, loss = 1.13829425
Iteration 25, loss = 0.18561608
Iteration 10, loss = 1.17602017
Iteration 7, loss = 1.23684244
Iteration 10, loss = 1.10860326
Iteration 26, loss = 0.17200765
Iteration 11, loss = 0.96977884
Iteration 8, loss = 1.11880224
Iteration 11, loss = 0.95316782
Iteration 27, loss = 0.20303338
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.453546, total=54.6min
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
Iteration 12, loss = 0.79055354
Iteration 9, loss = 1.13764332
Iteration 12, loss = 0.91366865
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
Iteration 13, loss = 1.01008562
Iteration 13, loss = 0.83933687
Iteration 10, loss = 1.16609022
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
Iteration 14, loss = 0.99013967
Iteration 14, loss = 0.94648700
Iteration 11, loss = 0.97381031
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
Iteration 15, loss = 0.80726660
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
Iteration 15, loss = 0.94371234
Iteration 12, loss = 1.03330951
Iteration 16, loss = 0.75382882
Iteration 13, loss = 0.96409428
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
Iteration 17, loss = 0.75344592
Iteration 14, loss = 1.02046678
Iteration 15, loss = 0.86178480
Iteration 18, loss = 0.62936871
Iteration 16, loss = 0.85272276
Iteration 19, loss = 0.52510154
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.426279, total=48.6min
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
Iteration 17, loss = 0.62666087
Iteration 20, loss = 0.79010335
Iteration 18, loss = 0.57811662
Iteration 21, loss = 0.77410593
Iteration 19, loss = 0.87792254
Iteration 22, loss = 0.60004803
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
Iteration 20, loss = 0.78227105
Iteration 21, loss = 0.70338344
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.437312, total=68.5min
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=adam .......
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.436747, total=72.4min
[Parallel(n_jobs=10)]: Done 12 tasks | elapsed: 262.2min
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=adam .......
Iteration 1, loss = 3.96248917
Iteration 2, loss = 1.73396836
Iteration 1, loss = 3.35051723
Iteration 3, loss = 1.56408818
Iteration 2, loss = 1.67170178
Iteration 4, loss = 1.43725087
Iteration 3, loss = 1.52481886
Iteration 5, loss = 1.31459791
Iteration 4, loss = 1.38661015
Iteration 6, loss = 1.18905685
Iteration 5, loss = 1.27324117
Iteration 7, loss = 1.07464205
Iteration 6, loss = 1.15235553
Iteration 8, loss = 0.95465074
Iteration 7, loss = 1.02818684
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=lbfgs, score=0.434870, total=267.0min
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=adam .......
Iteration 9, loss = 0.84931436
Iteration 8, loss = 0.91442027
Iteration 1, loss = 3.73348036
Iteration 10, loss = 0.77689212
Iteration 9, loss = 0.80876549
Iteration 2, loss = 1.67448134
Iteration 11, loss = 0.68153293
Iteration 10, loss = 0.67926821
Iteration 3, loss = 1.50395501
Iteration 12, loss = 0.61444938
Iteration 11, loss = 0.61375995
Iteration 4, loss = 1.35719969
Iteration 13, loss = 0.49061220
Iteration 12, loss = 0.56012757
Iteration 5, loss = 1.23872049
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=lbfgs, score=0.429288, total=49.6min
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=adam .......
Iteration 14, loss = 0.44434197
Iteration 13, loss = 0.47892402
Iteration 6, loss = 1.12143238
Iteration 15, loss = 0.42963297
Iteration 14, loss = 0.44453316
Iteration 1, loss = 3.48578469
Iteration 7, loss = 1.02773656
Iteration 16, loss = 0.41057686
Iteration 15, loss = 0.38434181
Iteration 2, loss = 1.68585602
Iteration 8, loss = 0.90539926
Iteration 17, loss = 0.34440376
Iteration 16, loss = 0.32632737
Iteration 3, loss = 1.52541861
Iteration 9, loss = 0.78403754
Iteration 18, loss = 0.31385553
Iteration 17, loss = 0.31966156
Iteration 4, loss = 1.39320050
Iteration 10, loss = 0.65382572
Iteration 19, loss = 0.24293375
Iteration 18, loss = 0.28515144
Iteration 5, loss = 1.26989454
Iteration 11, loss = 0.63801703
Iteration 20, loss = 0.21507224
Iteration 19, loss = 0.27546036
Iteration 6, loss = 1.18406886
Iteration 12, loss = 0.55521571
Iteration 21, loss = 0.18896400
Iteration 20, loss = 0.27394843
Iteration 7, loss = 1.07208232
Iteration 13, loss = 0.50957178
Iteration 22, loss = 0.19632566
Iteration 21, loss = 0.26040400
Iteration 8, loss = 0.94785589
Iteration 14, loss = 0.44738584
Iteration 23, loss = 0.24559544
Iteration 22, loss = 0.24038055
Iteration 9, loss = 0.79861689
Iteration 15, loss = 0.36049212
Iteration 24, loss = 0.28039386
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.447761, total=45.3min
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=adam .......
Iteration 23, loss = 0.20433077
Iteration 10, loss = 0.70531804
Iteration 16, loss = 0.32824225
Iteration 24, loss = 0.22447027
Iteration 11, loss = 0.68302504
Iteration 1, loss = 3.67568900
Iteration 17, loss = 0.31194656
Iteration 25, loss = 0.26074440
Iteration 12, loss = 0.55909487
Iteration 2, loss = 1.71489945
Iteration 18, loss = 0.29251655
Iteration 26, loss = 0.24027224
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.444666, total=48.6min
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=adam .......
Iteration 13, loss = 0.52248426
Iteration 3, loss = 1.55554573
Iteration 19, loss = 0.33266192
Iteration 1, loss = 3.54617795
Iteration 14, loss = 0.42178169
Iteration 4, loss = 1.41510183
Iteration 20, loss = 0.25673976
Iteration 2, loss = 1.73994029
Iteration 15, loss = 0.34770689
Iteration 5, loss = 1.30845887
Iteration 21, loss = 0.22370598
Iteration 3, loss = 1.66527464
Iteration 16, loss = 0.35923671
Iteration 6, loss = 1.16023482
Iteration 22, loss = 0.17441487
Iteration 4, loss = 1.54275339
Iteration 17, loss = 0.31147597
Iteration 7, loss = 1.05758262
Iteration 23, loss = 0.18738974
Iteration 5, loss = 1.49928038
Iteration 18, loss = 0.28436138
Iteration 8, loss = 0.96295383
Iteration 24, loss = 0.17800956
Iteration 19, loss = 0.23379718
Iteration 6, loss = 1.40909506
Iteration 9, loss = 0.85026990
Iteration 25, loss = 0.18375735
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.463074, total=44.2min
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=adam .......
Iteration 20, loss = 0.26956384
Iteration 7, loss = 1.41666827
Iteration 10, loss = 0.71673466
Iteration 1, loss = 3.36518246
Iteration 21, loss = 0.30466358
Iteration 8, loss = 1.27839763
Iteration 11, loss = 0.62961309
Iteration 2, loss = 1.81145088
Iteration 22, loss = 0.26258131
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
Iteration 9, loss = 1.11567295
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.447106, total=34.7min
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=adam .......
Iteration 12, loss = 0.58450925
Iteration 3, loss = 1.64287297
Iteration 10, loss = 1.47561848
Iteration 1, loss = 4.02844495
Iteration 13, loss = 0.47599935
Iteration 4, loss = 1.53614211
Iteration 11, loss = 1.37531804
Iteration 2, loss = 1.79637030
Iteration 14, loss = 0.41770751
Iteration 5, loss = 1.40404063
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=lbfgs, score=0.430569, total=107.9min
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=adam .......
Iteration 12, loss = 1.47082241
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.403403, total=18.9min
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=adam .......
Iteration 3, loss = 1.62297736
Iteration 15, loss = 0.42886164
Iteration 6, loss = 1.37032203
Iteration 1, loss = 3.56925623
Iteration 1, loss = 3.84772158
Iteration 4, loss = 1.49709602
Iteration 16, loss = 0.33430372
Iteration 7, loss = 1.29310147
Iteration 2, loss = 1.77448136
Iteration 2, loss = 1.75714810
Iteration 5, loss = 1.41705634
Iteration 17, loss = 0.29686078
Iteration 8, loss = 1.11809784
Iteration 3, loss = 1.59751882
Iteration 3, loss = 1.62572630
Iteration 6, loss = 1.40345911
Iteration 18, loss = 0.29596115
Iteration 9, loss = 1.02927585
Iteration 4, loss = 1.48249577
Iteration 7, loss = 1.34207888
Iteration 4, loss = 1.47044558
Iteration 19, loss = 0.27888974
Iteration 10, loss = 0.89368099
Iteration 5, loss = 1.45678223
Iteration 5, loss = 1.35099003
Iteration 8, loss = 1.33222222
Iteration 20, loss = 0.22657969
Iteration 11, loss = 1.65390440
Iteration 6, loss = 1.40543197
Iteration 21, loss = 0.25398174
Iteration 6, loss = 1.23423386
Iteration 9, loss = 1.16496029
Iteration 12, loss = 1.25037049
Iteration 7, loss = 1.35170020
Iteration 22, loss = 0.20402069
Iteration 7, loss = 1.28676051
Iteration 10, loss = 1.11357061
Iteration 13, loss = 1.26281227
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.357715, total=23.0min
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
Iteration 8, loss = 1.30264691
Iteration 23, loss = 0.24436018
Iteration 8, loss = 1.23909985
Iteration 11, loss = 1.00605112
Iteration 9, loss = 1.25928916
Iteration 24, loss = 0.22715525
Iteration 9, loss = 1.14523015
Iteration 12, loss = 0.95015198
Iteration 10, loss = 1.06003224
Iteration 25, loss = 0.22510200
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.459540, total=49.3min
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
Iteration 10, loss = 1.12625400
Iteration 13, loss = 0.90375728
Iteration 11, loss = 0.97976465
Iteration 11, loss = 1.19393566
Iteration 14, loss = 0.94556979
Iteration 12, loss = 1.01685459
Iteration 12, loss = 0.94908108
Iteration 15, loss = 0.66730472
Iteration 13, loss = 0.90266854
Iteration 13, loss = 0.78700681
Iteration 16, loss = 0.97861973
Iteration 14, loss = 0.89989633
Iteration 14, loss = 0.94258238
Iteration 17, loss = 0.72982338
Iteration 15, loss = 0.62400459
Iteration 15, loss = 1.05771527
Iteration 18, loss = 1.53050430
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
Iteration 16, loss = 1.26645662
Iteration 16, loss = 0.90295233
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
Iteration 17, loss = 1.00748595
Iteration 18, loss = 0.71764330
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=lbfgs, score=0.414756, total=27.9min
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.338014, total=57.3min
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.339017, total=53.6min
[CV] alpha=1e-05, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.434739, total=53.4min
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=lbfgs, score=0.439560, total=37.4min
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=lbfgs, score=0.422422, total=39.7min
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=lbfgs, score=0.435130, total=42.6min
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=lbfgs, score=0.437312, total=31.7min
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=lbfgs, score=0.427282, total=36.8min
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=lbfgs, score=0.429860, total=37.3min
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=adam .......
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=adam .......
Iteration 1, loss = 2.88417275
Iteration 1, loss = 2.95867359
Iteration 2, loss = 1.66457884
Iteration 3, loss = 1.50355525
Iteration 2, loss = 1.69235989
Iteration 4, loss = 1.37884546
Iteration 3, loss = 1.51038253
Iteration 5, loss = 1.24779193
Iteration 4, loss = 1.37358767
Iteration 6, loss = 1.13105864
Iteration 5, loss = 1.25842306
Iteration 7, loss = 1.03695733
Iteration 6, loss = 1.12791221
Iteration 8, loss = 0.91180590
Iteration 7, loss = 1.01880639
Iteration 9, loss = 0.77980504
Iteration 8, loss = 0.90234649
Iteration 10, loss = 0.66907496
Iteration 9, loss = 0.80579448
Iteration 11, loss = 0.61986207
Iteration 10, loss = 0.66110384
Iteration 12, loss = 0.55915620
Iteration 11, loss = 0.59822322
Iteration 13, loss = 0.46634046
Iteration 12, loss = 0.54204652
Iteration 14, loss = 0.36838048
Iteration 13, loss = 0.44655826
Iteration 15, loss = 0.38655448
Iteration 14, loss = 0.42438019
Iteration 16, loss = 0.32010492
Iteration 15, loss = 0.35138933
Iteration 17, loss = 0.28329576
Iteration 16, loss = 0.36386032
Iteration 18, loss = 0.25289232
Iteration 17, loss = 0.30311412
Iteration 19, loss = 0.25026107
Iteration 18, loss = 0.25496000
Iteration 20, loss = 0.30050519
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=lbfgs, score=0.433735, total=29.6min
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=adam .......
Iteration 19, loss = 0.17194342
Iteration 21, loss = 0.32366787
Iteration 1, loss = 2.97300705
Iteration 20, loss = 0.14590092
Iteration 22, loss = 0.25359500
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
Iteration 2, loss = 1.64881937
Iteration 21, loss = 0.19486374
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.454636, total=30.8min
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=adam .......
Iteration 3, loss = 1.49142553
Iteration 22, loss = 0.19182272
Iteration 1, loss = 2.99739752
Iteration 4, loss = 1.35556894
Iteration 23, loss = 0.20863011
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
Iteration 2, loss = 1.70095393
Iteration 5, loss = 1.22626102
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.448756, total=33.4min
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=adam .......
Iteration 3, loss = 1.52926749
Iteration 6, loss = 1.10210995
Iteration 1, loss = 2.75574216
Iteration 4, loss = 1.40807829
Iteration 7, loss = 1.01192922
Iteration 5, loss = 1.26639105
Iteration 2, loss = 1.64102053
Iteration 8, loss = 0.87337703
Iteration 6, loss = 1.15811150
Iteration 3, loss = 1.47979883
Iteration 9, loss = 0.75863985
Iteration 7, loss = 1.01217327
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=lbfgs, score=0.426866, total=155.3min
Iteration 4, loss = 1.35200278
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=adam .......
Iteration 10, loss = 0.64505800
Iteration 8, loss = 0.90810159
Iteration 5, loss = 1.20994861
Iteration 1, loss = 2.96866411
Iteration 11, loss = 0.58714236
Iteration 9, loss = 0.80698957
Iteration 6, loss = 1.08194756
Iteration 2, loss = 1.73491569
Iteration 12, loss = 0.52180280
Iteration 10, loss = 0.70875184
Iteration 7, loss = 0.96619692
Iteration 3, loss = 1.60420848
Iteration 13, loss = 0.47559493
Iteration 11, loss = 0.61313101
Iteration 8, loss = 0.84517601
Iteration 4, loss = 1.54971932
Iteration 14, loss = 0.37910432
Iteration 12, loss = 0.53415789
Iteration 9, loss = 0.72035952
Iteration 5, loss = 1.43702183
Iteration 15, loss = 0.31174271
Iteration 13, loss = 0.45893765
Iteration 10, loss = 0.65239109
Iteration 6, loss = 1.35528401
Iteration 16, loss = 0.27879615
Iteration 14, loss = 0.39997816
Iteration 11, loss = 0.55436688
Iteration 17, loss = 0.23583401
Iteration 7, loss = 1.18465082
Iteration 15, loss = 0.34848918
Iteration 12, loss = 0.48082961
Iteration 18, loss = 0.26047593
Iteration 8, loss = 1.05606773
Iteration 16, loss = 0.34544453
Iteration 13, loss = 0.45263640
Iteration 19, loss = 0.26782023
Iteration 9, loss = 0.94824029
Iteration 17, loss = 0.24420448
Iteration 14, loss = 0.36576661
Iteration 20, loss = 0.22942242
Iteration 10, loss = 1.19990349
Iteration 18, loss = 0.21120383
Iteration 15, loss = 0.33817017
Iteration 21, loss = 0.23336622
Iteration 11, loss = 0.87725424
Iteration 19, loss = 0.21272197
Iteration 16, loss = 0.25543758
Iteration 22, loss = 0.19915948
Iteration 12, loss = 1.12905784
Iteration 20, loss = 0.25842437
Iteration 17, loss = 0.23035899
Iteration 23, loss = 0.22426620
Iteration 13, loss = 0.93750096
Iteration 21, loss = 0.20864800
Iteration 18, loss = 0.23755076
Iteration 24, loss = 0.19789310
Iteration 14, loss = 0.85430559
Iteration 22, loss = 0.22051441
Iteration 19, loss = 0.23707541
Iteration 25, loss = 0.18822111
Iteration 15, loss = 0.81728444
Iteration 23, loss = 0.17617230
Iteration 20, loss = 0.24535409
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
Iteration 26, loss = 0.18486116
Iteration 16, loss = 0.61365834
Iteration 24, loss = 0.16746585
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.429570, total=23.7min
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=adam .......
Iteration 27, loss = 0.15741273
Iteration 17, loss = 0.43987506
Iteration 25, loss = 0.18753081
Iteration 1, loss = 2.81167653
Iteration 28, loss = 0.16078728
Iteration 18, loss = 0.96440434
Iteration 26, loss = 0.19833563
Iteration 2, loss = 1.74601108
Iteration 29, loss = 0.15885679
Iteration 19, loss = 1.03815732
Iteration 27, loss = 0.18611874
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.444112, total=29.5min
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=adam .......
Iteration 3, loss = 1.64177420
Iteration 30, loss = 0.08645354
Iteration 20, loss = 0.77188517
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.418418, total=24.1min
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=adam .......
Iteration 1, loss = 3.06038541
Iteration 4, loss = 1.50262400
Iteration 31, loss = 0.14470641
Iteration 1, loss = 2.86070011
Iteration 2, loss = 1.71348880
Iteration 5, loss = 1.37718926
Iteration 32, loss = 0.18036793
Iteration 2, loss = 1.70427702
Iteration 3, loss = 1.64902855
Iteration 6, loss = 1.27249606
Iteration 33, loss = 0.13418043
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
Iteration 3, loss = 1.55767323
Iteration 4, loss = 1.49095491
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.432136, total=36.5min
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=adam .......
Iteration 7, loss = 1.27169610
Iteration 4, loss = 1.48245880
Iteration 5, loss = 1.44314601
Iteration 8, loss = 1.54349010
Iteration 1, loss = 2.83726633
Iteration 5, loss = 1.40879547
Iteration 6, loss = 1.30648280
Iteration 9, loss = 1.25170752
Iteration 2, loss = 1.74439079
Iteration 6, loss = 1.29609460
Iteration 7, loss = 1.21731824
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=lbfgs, score=0.425425, total=531.9min
Iteration 10, loss = 1.29682940
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
Iteration 3, loss = 1.58291654
Iteration 8, loss = 1.15747249
Iteration 7, loss = 1.17893231
Iteration 11, loss = 1.05118754
Iteration 4, loss = 1.46957637
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=lbfgs, score=0.423881, total=534.7min
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
Iteration 9, loss = 1.20732814
Iteration 8, loss = 1.08432246
Iteration 12, loss = 1.33895336
Iteration 5, loss = 1.34414700
Iteration 10, loss = 1.14959240
Iteration 9, loss = 1.25161810
Iteration 13, loss = 1.34438512
Iteration 6, loss = 1.21300591
[CV] alpha=1e-05, hidden_layer_sizes=(3072, 3072), solver=lbfgs, score=0.435130, total=543.0min
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
Iteration 11, loss = 0.97456231
Iteration 14, loss = 1.04573330
Iteration 10, loss = 1.17793014
Iteration 7, loss = 1.33304173
Iteration 8, loss = 1.28098706
Iteration 11, loss = 1.06488746
Iteration 15, loss = 1.33278544
Iteration 12, loss = 0.78632889
Iteration 12, loss = 0.93610035
Iteration 9, loss = 1.19089118
Iteration 16, loss = 1.03863707
Iteration 13, loss = 1.05984830
Iteration 10, loss = 1.35194345
Iteration 13, loss = 0.97413507
Iteration 14, loss = 1.34233016
Iteration 17, loss = 1.03234308
Iteration 11, loss = 1.13972699
Iteration 14, loss = 0.92969348
Iteration 18, loss = 0.95815310
Iteration 15, loss = 1.07277400
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.415246, total=35.9min
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
Iteration 12, loss = 1.03567820
Iteration 15, loss = 0.84656892
Iteration 19, loss = 0.81525984
Iteration 13, loss = 1.07017026
Iteration 16, loss = 0.68526565
Iteration 20, loss = 0.71973651
Iteration 14, loss = 0.95915408
Iteration 21, loss = 0.64296986
Iteration 15, loss = 0.75709338
Iteration 22, loss = 0.56281256
Iteration 16, loss = 0.80830445
Iteration 17, loss = 0.98143526
Iteration 23, loss = 0.77732227
Iteration 17, loss = 0.57029997
Iteration 18, loss = 0.68498197
Iteration 24, loss = 0.74949308
Iteration 18, loss = 0.73267076
Iteration 19, loss = 0.66840275
Iteration 25, loss = 0.55913295
Iteration 19, loss = 0.68606972
Iteration 20, loss = 0.58257375
Iteration 26, loss = 0.35134888
Iteration 20, loss = 0.72344127
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
Iteration 21, loss = 0.45755877
Iteration 27, loss = 0.35812838
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.426707, total=56.8min
Iteration 22, loss = 0.39769908
Iteration 28, loss = 0.25058628
Iteration 23, loss = 0.30935679
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
Iteration 29, loss = 0.25235111
Iteration 24, loss = 0.70333325
Iteration 30, loss = 0.47715411
Iteration 25, loss = 0.39085534
Iteration 31, loss = 0.56390391
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
Iteration 26, loss = 0.30354038
Iteration 27, loss = 0.18035749
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.429860, total=71.1min
Iteration 28, loss = 0.13558713
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
Iteration 29, loss = 0.12907378
Iteration 30, loss = 0.13873241
Iteration 31, loss = 0.17053132
Iteration 32, loss = 0.11744277
Iteration 33, loss = 0.10068287
Iteration 34, loss = 0.99435202
Iteration 35, loss = 0.59743161
Iteration 36, loss = 0.58085724
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-06, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.409228, total=89.0min
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=lbfgs, score=0.444112, total=61.0min
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=lbfgs, score=0.443443, total=45.5min
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=lbfgs, score=0.446894, total=35.9min
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=lbfgs, score=0.449799, total=38.0min
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=adam .......
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=lbfgs, score=0.433300, total=45.1min
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=adam .......
Iteration 1, loss = 3.34980559
Iteration 1, loss = 3.96356395
Iteration 2, loss = 1.67619794
Iteration 2, loss = 1.73213997
Iteration 3, loss = 1.52166555
Iteration 3, loss = 1.56600963
Iteration 4, loss = 1.39534033
Iteration 4, loss = 1.42661440
Iteration 5, loss = 1.26209144
Iteration 5, loss = 1.31205931
Iteration 6, loss = 1.15353092
Iteration 6, loss = 1.17604622
Iteration 7, loss = 1.01353127
Iteration 7, loss = 1.05664894
Iteration 8, loss = 0.91115447
Iteration 8, loss = 0.92898637
Iteration 9, loss = 0.82007323
Iteration 9, loss = 0.84278410
Iteration 10, loss = 0.67571835
Iteration 10, loss = 0.75294563
Iteration 11, loss = 0.59627579
Iteration 11, loss = 0.63655432
Iteration 12, loss = 0.55253775
Iteration 12, loss = 0.56339828
Iteration 13, loss = 0.45251656
Iteration 13, loss = 0.48147709
Iteration 14, loss = 0.43059490
Iteration 14, loss = 0.41703244
Iteration 15, loss = 0.39870560
Iteration 15, loss = 0.37231608
Iteration 16, loss = 0.39938611
Iteration 16, loss = 0.36478252
Iteration 17, loss = 0.40976347
Iteration 17, loss = 0.37716439
Iteration 18, loss = 0.36504423
Iteration 18, loss = 0.34485310
Iteration 19, loss = 0.27193346
Iteration 19, loss = 0.29061913
Iteration 20, loss = 0.16795591
Iteration 20, loss = 0.24598504
Iteration 21, loss = 0.18971594
Iteration 21, loss = 0.24530552
Iteration 22, loss = 0.14871016
Iteration 22, loss = 0.21918059
Iteration 23, loss = 0.15739867
Iteration 23, loss = 0.22933781
Iteration 24, loss = 0.23394059
Iteration 24, loss = 0.18870255
Iteration 25, loss = 0.22858815
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.452642, total=29.3min
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=adam .......
Iteration 25, loss = 0.25333215
Iteration 26, loss = 0.20450604
Iteration 1, loss = 3.73430402
Iteration 27, loss = 0.17629885
Iteration 2, loss = 1.67240729
Iteration 28, loss = 0.16225728
Iteration 3, loss = 1.50347956
Iteration 29, loss = 0.11328346
Iteration 4, loss = 1.35711164
Iteration 30, loss = 0.12817469
Iteration 5, loss = 1.23340730
Iteration 31, loss = 0.14953600
Iteration 6, loss = 1.12356475
Iteration 32, loss = 0.19295405
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.448756, total=39.0min
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=adam .......
Iteration 7, loss = 1.04310319
Iteration 8, loss = 0.90785100
Iteration 1, loss = 3.48392504
Iteration 9, loss = 0.75446672
Iteration 2, loss = 1.68455784
Iteration 10, loss = 0.65655193
Iteration 3, loss = 1.51939467
Iteration 11, loss = 0.60694614
Iteration 4, loss = 1.39346329
Iteration 12, loss = 0.50370340
Iteration 5, loss = 1.26361239
Iteration 13, loss = 0.49020887
Iteration 6, loss = 1.16868206
Iteration 14, loss = 0.40947485
Iteration 7, loss = 1.07402891
Iteration 15, loss = 0.38631629
Iteration 8, loss = 0.92343885
Iteration 16, loss = 0.35452496
Iteration 9, loss = 0.79811191
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=lbfgs, score=0.428144, total=206.8min
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=adam .......
Iteration 17, loss = 0.30377399
Iteration 10, loss = 0.71037514
Iteration 18, loss = 0.25126649
Iteration 1, loss = 3.67426625
Iteration 11, loss = 0.67132439
Iteration 19, loss = 0.25142088
Iteration 2, loss = 1.71815653
Iteration 12, loss = 0.56121485
Iteration 20, loss = 0.24718292
Iteration 3, loss = 1.55364187
Iteration 13, loss = 0.51146464
Iteration 21, loss = 0.24911883
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=lbfgs, score=0.446660, total=227.8min
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=adam .......
Iteration 4, loss = 1.41943010
Iteration 14, loss = 0.44429893
Iteration 22, loss = 0.20174526
Iteration 5, loss = 1.29665326
Iteration 1, loss = 3.54529608
Iteration 15, loss = 0.39120109
Iteration 23, loss = 0.21137189
Iteration 6, loss = 1.16134787
Iteration 2, loss = 1.72486518
Iteration 16, loss = 0.35440622
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=lbfgs, score=0.418905, total=237.3min
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=adam .......
Iteration 24, loss = 0.20247762
Iteration 7, loss = 1.05825752
Iteration 3, loss = 1.67500733
Iteration 17, loss = 0.33246745
Iteration 1, loss = 3.36723557
Iteration 25, loss = 0.20214299
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.442116, total=49.8min
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=adam .......
Iteration 8, loss = 0.95304003
Iteration 4, loss = 1.52171862
Iteration 18, loss = 0.27727083
Iteration 2, loss = 1.77191833
Iteration 1, loss = 4.02829111
Iteration 9, loss = 0.82933104
Iteration 5, loss = 1.52164124
Iteration 19, loss = 0.27781647
Iteration 3, loss = 1.69238930
Iteration 10, loss = 0.70140500
Iteration 2, loss = 1.77875234
Iteration 6, loss = 1.41541744
Iteration 20, loss = 0.25921097
Iteration 4, loss = 1.53855398
Iteration 11, loss = 0.61231521
Iteration 3, loss = 1.61402380
Iteration 7, loss = 1.42189804
Iteration 21, loss = 0.23002769
Iteration 5, loss = 1.40434440
Iteration 12, loss = 0.55099181
Iteration 4, loss = 1.48936043
Iteration 8, loss = 1.29035913
Iteration 22, loss = 0.20505630
Iteration 6, loss = 1.40309191
Iteration 13, loss = 0.49786042
Iteration 5, loss = 1.40170211
Iteration 9, loss = 1.12646327
Iteration 23, loss = 0.15678558
Iteration 7, loss = 1.30893515
Iteration 14, loss = 0.46148675
Iteration 6, loss = 1.42363767
Iteration 10, loss = 1.43084737
Iteration 24, loss = 0.17929617
Iteration 8, loss = 1.15982756
Iteration 15, loss = 0.40869304
Iteration 7, loss = 1.35490493
Iteration 11, loss = 1.52938915
Iteration 25, loss = 0.18945607
Iteration 9, loss = 1.32391877
Iteration 16, loss = 0.39670381
Iteration 8, loss = 1.27835733
Iteration 12, loss = 1.43090114
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.389389, total=28.3min
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=adam .......
Iteration 26, loss = 0.19041056
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.464072, total=60.8min
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=adam .......
Iteration 10, loss = 1.36811090
Iteration 17, loss = 0.35415755
Iteration 9, loss = 1.19790761
Iteration 1, loss = 3.56984139
Iteration 1, loss = 3.84764984
Iteration 11, loss = 1.38320159
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.433868, total=25.8min
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
Iteration 18, loss = 0.28872359
Iteration 10, loss = 1.11624387
Iteration 2, loss = 1.80213004
Iteration 2, loss = 1.75761046
Iteration 19, loss = 0.28155961
Iteration 11, loss = 0.99218339
Iteration 3, loss = 1.60501085
Iteration 3, loss = 1.62850218
Iteration 20, loss = 0.24033602
Iteration 12, loss = 0.94618965
Iteration 4, loss = 1.47954345
Iteration 4, loss = 1.47270552
Iteration 21, loss = 0.21291033
Iteration 13, loss = 0.81699017
Iteration 5, loss = 1.45153177
Iteration 5, loss = 1.37009337
Iteration 22, loss = 0.19397538
Iteration 14, loss = 0.82633914
Iteration 6, loss = 1.36977565
Iteration 6, loss = 1.24032082
Iteration 23, loss = 0.19936622
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=lbfgs, score=0.425871, total=20.3min
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
Iteration 15, loss = 0.59585186
Iteration 7, loss = 1.32592304
Iteration 7, loss = 1.21591953
Iteration 24, loss = 0.19921849
Iteration 16, loss = 1.50743945
Iteration 8, loss = 1.25759732
Iteration 8, loss = 1.24049424
Iteration 25, loss = 0.19035871
Iteration 17, loss = 1.36061562
Iteration 9, loss = 1.28355608
Iteration 9, loss = 1.14204480
Iteration 26, loss = 0.22780406
Iteration 18, loss = 1.16421887
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.420261, total=56.4min
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
Iteration 10, loss = 1.05230432
Iteration 10, loss = 1.09329676
Iteration 27, loss = 0.24761011
Iteration 11, loss = 0.97996841
Iteration 11, loss = 1.19799266
Iteration 28, loss = 0.19909746
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.404595, total=86.4min
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=lbfgs, score=0.416750, total=22.7min
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
Iteration 12, loss = 0.96241264
Iteration 12, loss = 1.04153251
Iteration 13, loss = 0.90311109
Iteration 13, loss = 0.89771745
Iteration 14, loss = 1.03324559
Iteration 14, loss = 0.83584038
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=lbfgs, score=0.437126, total=27.5min
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
Iteration 15, loss = 0.69931056
Iteration 15, loss = 0.72410520
Iteration 16, loss = 1.36972482
Iteration 16, loss = 0.95555842
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=lbfgs, score=0.439122, total=28.9min
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
Iteration 17, loss = 1.06437287
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=lbfgs, score=0.442557, total=28.2min
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
Iteration 17, loss = 0.77253421
Iteration 18, loss = 0.88354021
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.404213, total=81.5min
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
Iteration 18, loss = 0.53270041
Iteration 19, loss = 0.71876919
Iteration 20, loss = 0.54873795
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=lbfgs, score=0.420420, total=30.8min
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=lbfgs ......
Iteration 21, loss = 0.42563870
Iteration 22, loss = 0.67504079
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=lbfgs, score=0.435872, total=32.6min
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=adam .......
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=lbfgs, score=0.431294, total=32.1min
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=adam .......
Iteration 1, loss = 2.95693254
Iteration 23, loss = 0.82386805
Iteration 1, loss = 2.88411655
Iteration 2, loss = 1.69161249
Iteration 2, loss = 1.66615732
Iteration 24, loss = 0.55332652
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-06, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.447791, total=111.8min
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=adam .......
Iteration 3, loss = 1.50507460
Iteration 3, loss = 1.50346252
Iteration 1, loss = 2.97299262
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=lbfgs, score=0.435306, total=33.1min
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=adam .......
Iteration 4, loss = 1.36585049
Iteration 4, loss = 1.37574246
Iteration 2, loss = 1.64875580
Iteration 5, loss = 1.24594534
Iteration 1, loss = 2.99731616
Iteration 5, loss = 1.24014745
Iteration 3, loss = 1.49080010
Iteration 6, loss = 1.12853043
Iteration 2, loss = 1.69135336
Iteration 6, loss = 1.12032207
Iteration 4, loss = 1.35468910
Iteration 7, loss = 1.00460235
Iteration 3, loss = 1.53197943
Iteration 7, loss = 1.01459356
Iteration 5, loss = 1.22343164
Iteration 8, loss = 0.89191825
Iteration 4, loss = 1.41662610
Iteration 8, loss = 0.89606030
Iteration 6, loss = 1.09559783
Iteration 5, loss = 1.27732222
Iteration 9, loss = 0.80294633
Iteration 9, loss = 0.78303489
Iteration 7, loss = 1.00919911
Iteration 6, loss = 1.16004057
Iteration 10, loss = 0.67662703
Iteration 10, loss = 0.67667285
Iteration 8, loss = 0.86563548
Iteration 7, loss = 1.03706483
Iteration 11, loss = 0.58970495
Iteration 11, loss = 0.58067407
Iteration 9, loss = 0.76487305
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=lbfgs, score=0.449799, total=31.2min
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=adam .......
Iteration 8, loss = 0.92005571
Iteration 12, loss = 0.49016042
Iteration 12, loss = 0.50895967
Iteration 10, loss = 0.65659825
Iteration 1, loss = 2.75250270
Iteration 9, loss = 0.80551355
Iteration 13, loss = 0.42787562
Iteration 13, loss = 0.48585005
Iteration 11, loss = 0.56559405
Iteration 2, loss = 1.64210556
Iteration 10, loss = 0.71418262
Iteration 14, loss = 0.39692157
Iteration 14, loss = 0.38170193
Iteration 12, loss = 0.49562857
Iteration 3, loss = 1.49020748
Iteration 11, loss = 0.63559025
Iteration 15, loss = 0.33951034
Iteration 15, loss = 0.34746974
Iteration 13, loss = 0.46345492
Iteration 4, loss = 1.34935431
Iteration 16, loss = 0.30428944
Iteration 12, loss = 0.55459267
Iteration 16, loss = 0.36351933
Iteration 14, loss = 0.36937705
Iteration 5, loss = 1.22119999
Iteration 17, loss = 0.30489927
Iteration 13, loss = 0.51093704
Iteration 17, loss = 0.26034438
Iteration 15, loss = 0.32742531
Iteration 6, loss = 1.09765401
Iteration 18, loss = 0.32389536
Iteration 14, loss = 0.40224213
Iteration 18, loss = 0.20854513
Iteration 16, loss = 0.29634947
Iteration 7, loss = 0.97924792
Iteration 19, loss = 0.26006806
Iteration 15, loss = 0.35640083
Iteration 19, loss = 0.22879935
Iteration 17, loss = 0.24949684
Iteration 8, loss = 0.85854380
Iteration 16, loss = 0.30447696
Iteration 20, loss = 0.19612956
Iteration 20, loss = 0.22365979
Iteration 18, loss = 0.22821401
Iteration 9, loss = 0.74228023
Iteration 21, loss = 0.16046449
Iteration 17, loss = 0.25657927
Iteration 21, loss = 0.17884209
Iteration 19, loss = 0.25587409
Iteration 10, loss = 0.65368896
Iteration 18, loss = 0.24139014
Iteration 22, loss = 0.13054717
Iteration 22, loss = 0.19675309
Iteration 20, loss = 0.21233572
Iteration 11, loss = 0.55725842
Iteration 23, loss = 0.13604186
Iteration 19, loss = 0.28161951
Iteration 23, loss = 0.27727247
Iteration 21, loss = 0.19245568
Iteration 12, loss = 0.47468440
Iteration 24, loss = 0.12688314
Iteration 20, loss = 0.24523971
Iteration 24, loss = 0.25960694
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.448654, total=36.8min
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=adam .......
Iteration 22, loss = 0.20990726
Iteration 13, loss = 0.45619250
Iteration 25, loss = 0.19636579
Iteration 21, loss = 0.23306200
Iteration 1, loss = 2.96705553
Iteration 23, loss = 0.24939775
Iteration 14, loss = 0.41251993
Iteration 26, loss = 0.24578893
Iteration 22, loss = 0.20678670
Iteration 2, loss = 1.72860350
Iteration 24, loss = 0.23194788
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.433134, total=36.2min
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=adam .......
Iteration 15, loss = 0.33926950
Iteration 27, loss = 0.20854668
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.460697, total=41.4min
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=adam .......
Iteration 23, loss = 0.22376172
Iteration 3, loss = 1.62562897
Iteration 1, loss = 2.81166171
Iteration 16, loss = 0.29340784
Iteration 1, loss = 3.05932833
Iteration 24, loss = 0.22853930
Iteration 4, loss = 1.57717702
Iteration 2, loss = 1.74549827
Iteration 17, loss = 0.24405119
Iteration 25, loss = 0.16014956
Iteration 2, loss = 1.72120409
Iteration 5, loss = 1.46810804
Iteration 18, loss = 0.23512580
Iteration 3, loss = 1.63981976
Iteration 26, loss = 0.18817189
Iteration 3, loss = 1.64929162
Iteration 6, loss = 1.45408542
Iteration 4, loss = 1.51275523
Iteration 19, loss = 0.23968953
Iteration 27, loss = 0.15290637
Iteration 4, loss = 1.48037469
Iteration 7, loss = 1.24748673
Iteration 20, loss = 0.20665614
Iteration 5, loss = 1.38611460
Iteration 28, loss = 0.12653333
Iteration 5, loss = 1.43817433
Iteration 8, loss = 1.13949699
Iteration 21, loss = 0.15772687
Iteration 6, loss = 1.28317843
Iteration 29, loss = 0.19595250
Iteration 6, loss = 1.34318743
Iteration 9, loss = 1.04942567
Iteration 22, loss = 0.16111205
Iteration 7, loss = 1.26712745
Iteration 30, loss = 0.18785899
Iteration 7, loss = 1.21655040
Iteration 10, loss = 1.10472890
Iteration 23, loss = 0.17194452
Iteration 8, loss = 1.50272571
Iteration 31, loss = 0.17771239
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.453094, total=45.3min
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=adam .......
Iteration 8, loss = 1.16944482
Iteration 11, loss = 0.85324429
Iteration 24, loss = 0.18566910
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.426573, total=33.9min
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=adam .......
Iteration 9, loss = 1.25150877
Iteration 1, loss = 2.86074871
Iteration 9, loss = 1.15580997
Iteration 12, loss = 1.10131138
Iteration 10, loss = 1.08206367
Iteration 1, loss = 2.83505171
Iteration 2, loss = 1.70079819
Iteration 10, loss = 1.16363908
Iteration 13, loss = 0.88516597
Iteration 11, loss = 0.99411808
Iteration 2, loss = 1.74123806
Iteration 3, loss = 1.56243654
Iteration 11, loss = 1.01131362
Iteration 14, loss = 0.80527297
Iteration 12, loss = 1.09277526
Iteration 3, loss = 1.60301138
Iteration 4, loss = 1.48241317
Iteration 12, loss = 0.77991214
Iteration 15, loss = 0.68918991
Iteration 13, loss = 1.23320347
Iteration 4, loss = 1.48188894
Iteration 5, loss = 1.45906443
Iteration 13, loss = 0.89389837
Iteration 16, loss = 0.57821852
Iteration 14, loss = 0.99322468
Iteration 5, loss = 1.35595147
Iteration 6, loss = 1.32137760
Iteration 14, loss = 1.17791993
Iteration 17, loss = 0.43574444
Iteration 15, loss = 1.20281100
Iteration 6, loss = 1.21833367
Iteration 7, loss = 1.16947138
Iteration 15, loss = 1.01502341
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.424273, total=21.6min
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
Iteration 18, loss = 0.75700691
Iteration 16, loss = 0.96515839
Iteration 7, loss = 1.19887064
Iteration 8, loss = 1.08177692
Iteration 19, loss = 0.60582763
Iteration 8, loss = 1.26782199
Iteration 17, loss = 0.96519461
Iteration 9, loss = 1.19542894
Iteration 20, loss = 0.39140597
Iteration 9, loss = 1.14849230
Iteration 18, loss = 0.65552456
Iteration 10, loss = 1.02007614
Iteration 21, loss = 0.27366796
Iteration 10, loss = 1.21609131
Iteration 19, loss = 0.68870719
Iteration 11, loss = 1.06292592
Iteration 22, loss = 0.24677466
Iteration 11, loss = 1.00102797
Iteration 20, loss = 0.71773485
Iteration 12, loss = 0.95215724
Iteration 23, loss = 0.22262456
Iteration 12, loss = 1.04978221
Iteration 21, loss = 0.61876025
Iteration 13, loss = 0.86700913
Iteration 24, loss = 0.19487498
Iteration 13, loss = 1.05061732
Iteration 22, loss = 0.96037320
Iteration 14, loss = 0.88238819
Iteration 25, loss = 0.14119529
Iteration 14, loss = 1.11634389
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.429719, total=29.8min
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
Iteration 23, loss = 1.04157572
Iteration 15, loss = 0.75509753
Iteration 26, loss = 0.17088580
Iteration 24, loss = 1.03411821
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.441884, total=46.5min
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
Iteration 16, loss = 0.75743425
Iteration 27, loss = 0.16196255
Iteration 17, loss = 0.87358400
Iteration 28, loss = 0.19515397
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.406406, total=56.5min
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
Iteration 18, loss = 0.52278446
Iteration 19, loss = 0.60687350
Iteration 20, loss = 0.89111581
Iteration 21, loss = 0.59474305
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-07, hidden_layer_sizes=(3072, 3072), solver=adam, score=0.414243, total=55.2min
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=lbfgs, score=0.416915, total=59.7min
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=lbfgs, score=0.445110, total=45.8min
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=lbfgs, score=0.443443, total=41.2min
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=lbfgs, score=0.444890, total=28.9min
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=lbfgs, score=0.428285, total=47.9min
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=lbfgs, score=0.450351, total=51.3min
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=lbfgs ......
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=adam .......
Iteration 1, loss = 3.96362793
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=lbfgs, score=0.433566, total=112.2min
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=adam .......
Iteration 2, loss = 1.73167346
Iteration 1, loss = 3.34982581
Iteration 3, loss = 1.56733820
Iteration 2, loss = 1.67610144
Iteration 4, loss = 1.43057255
Iteration 3, loss = 1.52525204
Iteration 5, loss = 1.31572901
Iteration 4, loss = 1.39510871
Iteration 6, loss = 1.17773277
Iteration 5, loss = 1.27428120
Iteration 7, loss = 1.05556223
Iteration 6, loss = 1.15826989
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=lbfgs, score=0.439681, total=156.6min
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=adam .......
Iteration 8, loss = 0.95173651
Iteration 7, loss = 1.03666098
Iteration 1, loss = 3.73351008
Iteration 9, loss = 0.84390321
Iteration 8, loss = 0.90997103
Iteration 10, loss = 0.76217873
Iteration 2, loss = 1.67144107
Iteration 9, loss = 0.83593259
Iteration 11, loss = 0.69088003
Iteration 3, loss = 1.50501733
Iteration 10, loss = 0.68032012
Iteration 12, loss = 0.58957477
Iteration 4, loss = 1.35815776
Iteration 11, loss = 0.62619238
Iteration 13, loss = 0.51066361
Iteration 5, loss = 1.23776324
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=lbfgs, score=0.451807, total=33.4min
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=adam .......
Iteration 12, loss = 0.59998443
Iteration 6, loss = 1.13331041
Iteration 14, loss = 0.44637195
Iteration 1, loss = 3.48490195
Iteration 13, loss = 0.51111835
Iteration 7, loss = 1.02903180
Iteration 15, loss = 0.37708478
Iteration 2, loss = 1.68212338
Iteration 14, loss = 0.43765541
Iteration 16, loss = 0.36271229
Iteration 8, loss = 0.88459738
Iteration 15, loss = 0.37986964
Iteration 3, loss = 1.52563153
Iteration 9, loss = 0.78628826
Iteration 17, loss = 0.31498140
Iteration 4, loss = 1.38872660
Iteration 16, loss = 0.34077712
Iteration 18, loss = 0.29120079
Iteration 10, loss = 0.67408537
Iteration 5, loss = 1.26429956
Iteration 17, loss = 0.35490048
Iteration 19, loss = 0.29981912
Iteration 11, loss = 0.65971216
Iteration 6, loss = 1.17096045
Iteration 18, loss = 0.32549914
Iteration 20, loss = 0.29750147
Iteration 12, loss = 0.52740437
Iteration 7, loss = 1.06667829
Iteration 19, loss = 0.28179503
Iteration 21, loss = 0.21903707
Iteration 13, loss = 0.47173434
Iteration 8, loss = 0.93002755
Iteration 20, loss = 0.22238722
Iteration 22, loss = 0.22634408
Iteration 14, loss = 0.41820011
Iteration 9, loss = 0.78631931
Iteration 21, loss = 0.20872217
Iteration 23, loss = 0.25301503
Iteration 15, loss = 0.36566089
Iteration 10, loss = 0.68965844
Iteration 22, loss = 0.22689621
Iteration 16, loss = 0.31553528
Iteration 24, loss = 0.23402475
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.430846, total=50.2min
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=adam .......
Iteration 11, loss = 0.65266134
Iteration 23, loss = 0.20580421
Iteration 17, loss = 0.28824507
Iteration 1, loss = 3.67407161
Iteration 12, loss = 0.56123632
Iteration 24, loss = 0.21229727
Iteration 18, loss = 0.27212587
Iteration 2, loss = 1.71775936
Iteration 13, loss = 0.51814884
Iteration 25, loss = 0.25756427
Iteration 19, loss = 0.29818187
Iteration 3, loss = 1.55444863
Iteration 26, loss = 0.20466374
Iteration 14, loss = 0.41341149
Iteration 20, loss = 0.20288495
Iteration 4, loss = 1.41557716
Iteration 27, loss = 0.17779408
Iteration 15, loss = 0.36218387
Iteration 21, loss = 0.15627271
Iteration 5, loss = 1.29860516
Iteration 28, loss = 0.18981369
Iteration 16, loss = 0.35333305
Iteration 22, loss = 0.16634441
Iteration 6, loss = 1.15501262
Iteration 29, loss = 0.16436753
Iteration 17, loss = 0.29369208
Iteration 23, loss = 0.17173446
Iteration 7, loss = 1.05203011
Iteration 30, loss = 0.22652306
Iteration 18, loss = 0.23839616
Iteration 24, loss = 0.19165427
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.440120, total=42.1min
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=adam .......
Iteration 8, loss = 0.95919110
Iteration 31, loss = 0.22078350
Iteration 19, loss = 0.24109026
Iteration 1, loss = 3.54527872
Iteration 9, loss = 0.84698266
Iteration 32, loss = 0.18544830
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
Iteration 20, loss = 0.27839069
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.461615, total=61.3min
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=adam .......
Iteration 2, loss = 1.72493752
Iteration 10, loss = 0.70228932
Iteration 21, loss = 0.24983266
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.442116, total=34.0min
Iteration 1, loss = 3.36963847
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=adam .......
Iteration 3, loss = 1.67499894
Iteration 11, loss = 0.63204558
Iteration 2, loss = 1.74264311
Iteration 1, loss = 4.02834048
Iteration 4, loss = 1.52715406
Iteration 12, loss = 0.55681559
Iteration 3, loss = 1.70141487
Iteration 2, loss = 1.78247490
Iteration 5, loss = 1.54045854
Iteration 13, loss = 0.51098674
Iteration 4, loss = 1.51652928
Iteration 3, loss = 1.61434274
Iteration 14, loss = 0.41214865
Iteration 6, loss = 1.42818893
Iteration 5, loss = 1.39858824
Iteration 4, loss = 1.48999085
Iteration 15, loss = 0.37669416
Iteration 7, loss = 1.49256529
Iteration 6, loss = 1.38120002
Iteration 5, loss = 1.40763180
Iteration 16, loss = 0.37373764
Iteration 8, loss = 1.29403702
Iteration 7, loss = 1.31893290
Iteration 17, loss = 0.34398853
Iteration 6, loss = 1.39348176
Iteration 9, loss = 1.15111505
Iteration 8, loss = 1.13207583
Iteration 18, loss = 0.31131223
Iteration 7, loss = 1.31444631
Iteration 10, loss = 1.35504046
Iteration 9, loss = 0.99673439
Iteration 19, loss = 0.24256963
Iteration 8, loss = 1.24045987
Iteration 11, loss = 1.40290523
Iteration 20, loss = 0.25761797
Iteration 10, loss = 1.01651344
Iteration 9, loss = 1.11151791
Iteration 12, loss = 1.64954375
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.378378, total=19.0min
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=adam .......
Iteration 21, loss = 0.19432833
Iteration 11, loss = 1.23241245
Iteration 10, loss = 1.00235916
Iteration 1, loss = 3.56802792
Iteration 22, loss = 0.20711981
Iteration 12, loss = 1.02218648
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.407816, total=18.8min
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=adam .......
Iteration 11, loss = 0.92390140
Iteration 2, loss = 1.78230329
Iteration 23, loss = 0.18919988
Iteration 1, loss = 3.84722479
Iteration 12, loss = 0.95550290
Iteration 3, loss = 1.61584120
Iteration 24, loss = 0.22939504
Iteration 2, loss = 1.75807111
Iteration 13, loss = 0.74860108
Iteration 4, loss = 1.48693222
Iteration 25, loss = 0.21569332
Iteration 3, loss = 1.62637107
Iteration 14, loss = 0.80137090
Iteration 5, loss = 1.44497614
Iteration 26, loss = 0.20058921
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.412587, total=40.6min
Iteration 4, loss = 1.47845178
Iteration 15, loss = 0.54482595
Iteration 6, loss = 1.39051737
Iteration 5, loss = 1.37038025
Iteration 16, loss = 1.11634132
Iteration 7, loss = 1.30830869
Iteration 6, loss = 1.25660201
Iteration 17, loss = 0.75191710
Iteration 8, loss = 1.31799682
Iteration 7, loss = 1.26103104
Iteration 18, loss = 0.60451718
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.435306, total=27.4min
Iteration 9, loss = 1.39591821
Iteration 8, loss = 1.25088348
Iteration 10, loss = 1.17197747
Iteration 9, loss = 1.16658698
Iteration 11, loss = 1.02339773
Iteration 10, loss = 1.12042400
Iteration 12, loss = 0.99088739
Iteration 11, loss = 1.15223259
Iteration 13, loss = 0.90745897
Iteration 12, loss = 0.95183135
Iteration 14, loss = 1.01030243
Iteration 13, loss = 0.94336698
Iteration 15, loss = 0.74675362
Iteration 14, loss = 0.81157825
Iteration 16, loss = 0.99112104
Iteration 15, loss = 0.97484033
Iteration 17, loss = 0.85448865
Iteration 16, loss = 0.80679910
Iteration 18, loss = 0.73612666
Iteration 17, loss = 0.65909326
Iteration 19, loss = 1.30849838
Iteration 18, loss = 0.45831179
Iteration 20, loss = 1.12600719
Iteration 19, loss = 0.85017106
Iteration 21, loss = 0.92372148
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.427282, total=25.5min
Iteration 20, loss = 0.58507553
Iteration 21, loss = 0.56336295
Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
[CV] alpha=1e-07, hidden_layer_sizes=(4000, 4000), solver=adam, score=0.410643, total=24.1min
In [ ]:
print('\n'.join('{}: {}'.format(k, v) for k, v in estimator.cv_results_.items())); print()
print('Scores for each set of parameters')
print('\n'.join([str(param) for param in estimator.grid_scores_])); print()
print('Best score')
print(estimator.best_score_); print()
print('Parameters corresponding to best score')
print(estimator.best_params_); print()
In [ ]:
means = estimator_result.cv_results_['mean_test_score']
stds = estimator_result.cv_results_['std_test_score']
params = estimator_result.cv_results_['params']
for mean, stdev, param in zip(means, stds, params):
print('%f (%f) with: %r' % (mean, stdev, param))
In [ ]:
start = time.time()
file_utils.mkdir_p(output_directory)
model_output_path = output_directory + '07_' + param_name + '.pkl'
joblib.dump(estimator, model_output_path)
end = time.time()
print('[ Step 4] Write obtained model to %s in %.6f ms' %(model_output_path, ((end-start)*1000)))
total_time += (end-start)
In [ ]:
scores = []
exp = []
dict_key = ['train', 'test']
for key in dict_key:
scores_list = []
for i in range(num_folds):
key_name = 'split' + str(i) + '_' + key + '_score'
scores_list.append(estimator.cv_results_[key_name].tolist())
scores.append(scores_list)
scores = np.array(scores)
means = np.mean(np.array(scores).T, axis=1)
stds = np.std(np.array(scores).T, axis=1)
In [ ]:
plot_tools.plot_variance(scores, means, stds, legend=['Training data', 'Validation data'],
plot_title=['Training scores for best parameters for MLP using raw pixels in CIFAR 10',
'Validation scores for best parameters for MLP using raw pixels in CIFAR 10'],
fig_size=(800,600), plot_xlabel=['MLP Parameters','MLP Parameters'],
plot_ylabel=['Training accuracy of the model','Validation accuracy of the model'], plot_lib='bokeh',
matplotlib_style='default', bokeh_notebook=True)
In [ ]:
start = time.time()
prediction_numbers = estimator.predict(test_images)
prediction_classes = []
num_test_images = test_images.shape[0]
for i in range(num_test_images):
prediction_classes.append(cifar10.classes[int(prediction_numbers[i])])
end = time.time()
print('[ Step 9] Make prediction on test dataset in %.6f ms' %((end-start)*1000))
total_time += (end-start)
In [ ]:
cifar10.plot_images(cifar10.test.data[:50], cifar10.test.class_names[:50], cls_pred=prediction_classes[:50],
nrows=5, ncols=10, fig_size=(20,50), fontsize=35, convert=True)
In [ ]:
start = time.time()
plot_tools.plot_confusion_matrix(cifar10.test.class_labels, prediction_numbers, classes=cifar10.classes,
normalize=True, title='Confusion matrix for test set')
print(skm.classification_report(cifar10.test.class_labels, prediction_numbers, target_names=cifar10.classes))
test_accuracy = skm.accuracy_score(cifar10.test.class_labels, prediction_numbers, normalize=True)
print('Accuracy score on test data: ' + str(test_accuracy))
end = time.time()
total_time += (end-start)
In [ ]:
start = time.time()
print('Prediction done on %d images' %cifar10.test.data.shape[0])
print('Accuracy of the classifier: %.4f' %estimator.score(test_images, cifar10.test.class_labels))
end = time.time()
total_time += (end-start)
In [ ]:
start = time.time()
indices = np.arange(1, test_images.shape[0]+1)
predictions = np.column_stack((indices, prediction_classes))
file_utils.mkdir_p(output_directory)
output_csv_file = output_directory + '07_' + param_name + '.csv'
column_names = ['id', 'label']
predict_test_df = pd.DataFrame(data=predictions, columns=column_names)
predict_test_df.to_csv(output_csv_file, index=False)
end = time.time()
print('[ Step 6] Writing the test data to file: %s in %.6f ms' %(output_csv_file, (end-start)*1000))
total_time += (end-start)
In [ ]:
def output_HTML(read_file, output_file):
from nbconvert import HTMLExporter
import codecs
import nbformat
exporter = HTMLExporter()
output_notebook = nbformat.read(read_file, as_version=4)
print()
output, resources = exporter.from_notebook_node(output_notebook)
codecs.open(output_file, 'w', encoding='utf-8').write(output)
In [ ]:
%%javascript
var notebook = IPython.notebook
notebook.save_notebook()
In [ ]:
%%javascript
var kernel = IPython.notebook.kernel;
var thename = window.document.getElementById("notebook_name").innerHTML;
var command = "theNotebook = " + "'"+thename+"'";
kernel.execute(command);
In [ ]:
current_file = './' + theNotebook + '.ipynb'
output_file = output_directory + str(file_no).zfill(2) + '_exp_no_' + str(exp_no) + '_' + theNotebook + '.html'
print('Current file: ' + str(current_file))
print('Output file: ' + str(output_file))
file_utils.mkdir_p(output_directory)
output_HTML(current_file, output_file)
In [ ]:
print('Code took %.6f s to run on training with %d examples' % (total_time,num_train_images))
In [ ]:
Content source: sonapraneeth-a/object-classification
Similar notebooks: