In [1]:

    

from hw_utils import *
from copy import copy


    

    




Using Theano backend.

In [2]:

    

# Variables

data_path = '../data/MiniBooNE_PID.txt'


    

In [3]:

    

## (c)
X_tr, Y_tr, X_te, Y_te = loaddata(data_path)
print "Train X, Y :", X_tr.shape, Y_tr.shape
print "Test  X, Y :", X_te.shape, Y_te.shape
X_tr, X_te = normalize(X_tr, X_te)
d_in = X_tr.shape[1] # input features
d_out = Y_tr.shape[1] # Output predictions


    

    




Train X, Y : (104051, 50) (104051, 2)
Test  X, Y : (26013, 50) (26013, 2)






    




/Users/thammegr/anaconda/lib/python2.7/site-packages/keras/utils/np_utils.py:14: VisibleDeprecationWarning: using a non-integer number instead of an integer will result in an error in the future
  Y = np.zeros((len(y), nb_classes))
/Users/thammegr/anaconda/lib/python2.7/site-packages/keras/utils/np_utils.py:16: VisibleDeprecationWarning: using a non-integer number instead of an integer will result in an error in the future
  Y[i, y[i]] = 1.

In [4]:

    

## (d)

# default args
args = {
    'actfn':'linear',
    'last_act':'softmax',
    'reg_coeffs': [0.0],
    'num_epoch': 30,
    'batch_size': 1000,
    'sgd_lr': 0.001,
    'sgd_decays': [0.0],
    'sgd_moms': [0.0],
    'sgd_Nesterov': False,
    'EStop': False,
    'verbose': False
}

# FIXME: BEGIN : For quick testing.. Remove this later
args['batch_size'] = 10
X_tr, Y_tr = X_tr[0:100], Y_tr[:100]
X_te, Y_te = X_te[0:20], Y_te[:20]
print "Train X, Y :", X_tr.shape, Y_tr.shape

# FIXME: END

print("\n\n## (d) Linear Activation ")
archs = [[d_in, d_out], [d_in, 50, d_out], [d_in, 50, 50, d_out], [d_in, 50, 50, 50, d_out]]
res = testmodels(X_tr, Y_tr, X_te, Y_te, archs, **args)

archs = [[d_in, 50, d_out], [d_in, 500, d_out], [d_in, 500, 300, d_out],
         [d_in, 800, 500, 300, d_out], [d_in, 800, 800, 500, 300, d_out]]
res = testmodels(X_tr, Y_tr, X_te, Y_te, archs, **args)


    

    




Train X, Y : (100, 50) (100, 2)


## (d) Linear Activation 
architecture=[50, 2], lambda=0.0, decay=0.0, momentum=0.0, actfn=linear: score=0.65000000596 | time=1.86916589737
architecture=[50, 50, 2], lambda=0.0, decay=0.0, momentum=0.0, actfn=linear: score=0.90000000596 | time=0.925344944
architecture=[50, 50, 50, 2], lambda=0.0, decay=0.0, momentum=0.0, actfn=linear: score=0.799999982119 | time=1.17178320885
architecture=[50, 50, 50, 50, 2], lambda=0.0, decay=0.0, momentum=0.0, actfn=linear: score=0.84999999404 | time=1.22634387016
Best Config: architecture = [50, 50, 2], lambda = 0.0, decay = 0.0, momentum = 0.0, actfn = linear, best_acc = 0.90000000596
Mean Time = 1.29815948009seconds, |Models| = 4, Total Time = 5.19263792038seconds
architecture=[50, 50, 2], lambda=0.0, decay=0.0, momentum=0.0, actfn=linear: score=0.90000000596 | time=1.06496405602
architecture=[50, 500, 2], lambda=0.0, decay=0.0, momentum=0.0, actfn=linear: score=0.84999999404 | time=0.992525100708
architecture=[50, 500, 300, 2], lambda=0.0, decay=0.0, momentum=0.0, actfn=linear: score=0.899999976158 | time=1.31494212151
architecture=[50, 800, 500, 300, 2], lambda=0.0, decay=0.0, momentum=0.0, actfn=linear: score=0.899999976158 | time=1.4492418766
architecture=[50, 800, 800, 500, 300, 2], lambda=0.0, decay=0.0, momentum=0.0, actfn=linear: score=0.899999976158 | time=2.16674518585
Best Config: architecture = [50, 50, 2], lambda = 0.0, decay = 0.0, momentum = 0.0, actfn = linear, best_acc = 0.90000000596
Mean Time = 1.39768366814seconds, |Models| = 5, Total Time = 6.98841834068seconds

In [5]:

    

# (e)
print("\n\n## (e) Sigmoid Activation")
args['actfn'] = 'sigmoid'
_ = testmodels(X_tr, Y_tr, X_te, Y_te, archs, **args)


    

    





## (e) Sigmoid Activation
architecture=[50, 50, 2], lambda=0.0, decay=0.0, momentum=0.0, actfn=sigmoid: score=0.59999999404 | time=0.977607965469
architecture=[50, 500, 2], lambda=0.0, decay=0.0, momentum=0.0, actfn=sigmoid: score=0.59999999404 | time=1.02421379089
architecture=[50, 500, 300, 2], lambda=0.0, decay=0.0, momentum=0.0, actfn=sigmoid: score=0.59999999404 | time=1.30875492096
architecture=[50, 800, 500, 300, 2], lambda=0.0, decay=0.0, momentum=0.0, actfn=sigmoid: score=0.59999999404 | time=1.70223593712
architecture=[50, 800, 800, 500, 300, 2], lambda=0.0, decay=0.0, momentum=0.0, actfn=sigmoid: score=0.59999999404 | time=2.83162212372
Best Config: architecture = [50, 50, 2], lambda = 0.0, decay = 0.0, momentum = 0.0, actfn = sigmoid, best_acc = 0.59999999404
Mean Time = 1.56888694763seconds, |Models| = 5, Total Time = 7.84443473816seconds

In [6]:

    

# (f)
print("\n\n## (f) ReLu Activation")
args['actfn'] = 'relu'
args['sgd_lr'] = 5e-4
_ = testmodels(X_tr, Y_tr, X_te, Y_te, archs, **args)


    

    





## (f) ReLu Activation
architecture=[50, 50, 2], lambda=0.0, decay=0.0, momentum=0.0, actfn=relu: score=0.550000011921 | time=1.23269987106
architecture=[50, 500, 2], lambda=0.0, decay=0.0, momentum=0.0, actfn=relu: score=0.84999999404 | time=0.984317064285
architecture=[50, 500, 300, 2], lambda=0.0, decay=0.0, momentum=0.0, actfn=relu: score=0.800000011921 | time=1.41553783417
architecture=[50, 800, 500, 300, 2], lambda=0.0, decay=0.0, momentum=0.0, actfn=relu: score=0.700000017881 | time=1.92823719978
architecture=[50, 800, 800, 500, 300, 2], lambda=0.0, decay=0.0, momentum=0.0, actfn=relu: score=0.59999999404 | time=2.5575799942
Best Config: architecture = [50, 500, 2], lambda = 0.0, decay = 0.0, momentum = 0.0, actfn = relu, best_acc = 0.84999999404
Mean Time = 1.6236743927seconds, |Models| = 5, Total Time = 8.1183719635seconds

In [7]:

    

# (g)
print("\n\n## (g) Regularization Coefficients")
archs = [[d_in, 800, 500, 300, d_out]]
args['reg_coeffs'] = [1e-7, 5e-7, 1e-6, 5e-6, 1e-5]
best, _ = testmodels(X_tr, Y_tr, X_te, Y_te, archs, **args)
best_lambda_noEstop = best[1]
print("Best Regularization Coefficient=", best_lambda_noEstop)


    

    





## (g) Regularization Coefficients
architecture=[50, 800, 500, 300, 2], lambda=1e-07, decay=0.0, momentum=0.0, actfn=relu: score=0.65000000596 | time=3.26436805725
architecture=[50, 800, 500, 300, 2], lambda=5e-07, decay=0.0, momentum=0.0, actfn=relu: score=0.59999999404 | time=3.47442913055
architecture=[50, 800, 500, 300, 2], lambda=1e-06, decay=0.0, momentum=0.0, actfn=relu: score=0.59999999404 | time=3.42006278038
architecture=[50, 800, 500, 300, 2], lambda=5e-06, decay=0.0, momentum=0.0, actfn=relu: score=0.700000017881 | time=3.3063929081
architecture=[50, 800, 500, 300, 2], lambda=1e-05, decay=0.0, momentum=0.0, actfn=relu: score=0.59999999404 | time=3.420814991
Best Config: architecture = [50, 800, 500, 300, 2], lambda = 5e-06, decay = 0.0, momentum = 0.0, actfn = relu, best_acc = 0.700000017881
Mean Time = 3.37721357346seconds, |Models| = 5, Total Time = 16.8860678673seconds
('Best Regularization Coefficient=', 5e-06)

In [8]:

    

# (h)
print("\n\n## (h) Regularization Coefficients -- Early stop")
args['EStop'] = True
best, _ = testmodels(X_tr, Y_tr, X_te, Y_te, archs, **args)
best_lambda_EStop = best[1]
print("Best Regularization Coefficient with early stopping=", best_lambda_EStop)


    

    





## (h) Regularization Coefficients -- Early stop
Epoch 00022: early stopping
architecture=[50, 800, 500, 300, 2], lambda=1e-07, decay=0.0, momentum=0.0, actfn=relu: score=0.800000011921 | time=2.89277601242
Epoch 00007: early stopping
architecture=[50, 800, 500, 300, 2], lambda=5e-07, decay=0.0, momentum=0.0, actfn=relu: score=0.59999999404 | time=2.30634999275
Epoch 00007: early stopping
architecture=[50, 800, 500, 300, 2], lambda=1e-06, decay=0.0, momentum=0.0, actfn=relu: score=0.800000011921 | time=2.31239390373
Epoch 00020: early stopping
architecture=[50, 800, 500, 300, 2], lambda=5e-06, decay=0.0, momentum=0.0, actfn=relu: score=0.799999982119 | time=2.92426681519
Epoch 00007: early stopping
architecture=[50, 800, 500, 300, 2], lambda=1e-05, decay=0.0, momentum=0.0, actfn=relu: score=0.59999999404 | time=2.315128088
Best Config: architecture = [50, 800, 500, 300, 2], lambda = 1e-07, decay = 0.0, momentum = 0.0, actfn = relu, best_acc = 0.800000011921
Mean Time = 2.55018296242seconds, |Models| = 5, Total Time = 12.7509148121seconds
('Best Regularization Coefficient with early stopping=', 1e-07)

In [9]:

    

# (i)
print("\n\n## (i) SGD Decay")
args['reg_coeffs'] = [5e-7]
args['num_epoch'] = 100
args['sgd_lr'] = 1e-5
args['sgd_decays'] = [1e-5, 5e-5, 1e-4, 3e-4, 7e-4, 1e-3]
args['EStop'] = False
best, _ = testmodels(X_tr, Y_tr, X_te, Y_te, archs, **args)
best_decay = best[2]
print("Best Decay", best_decay)


    

    





## (i) SGD Decay
architecture=[50, 800, 500, 300, 2], lambda=5e-07, decay=1e-05, momentum=0.0, actfn=relu: score=0.59999999404 | time=6.84515190125
architecture=[50, 800, 500, 300, 2], lambda=5e-07, decay=5e-05, momentum=0.0, actfn=relu: score=0.75 | time=8.42867708206
architecture=[50, 800, 500, 300, 2], lambda=5e-07, decay=0.0001, momentum=0.0, actfn=relu: score=0.799999982119 | time=6.8960351944
architecture=[50, 800, 500, 300, 2], lambda=5e-07, decay=0.0003, momentum=0.0, actfn=relu: score=0.350000008941 | time=6.87825584412
architecture=[50, 800, 500, 300, 2], lambda=5e-07, decay=0.0007, momentum=0.0, actfn=relu: score=0.550000011921 | time=6.68029594421
architecture=[50, 800, 500, 300, 2], lambda=5e-07, decay=0.001, momentum=0.0, actfn=relu: score=0.45000000298 | time=6.56511807442
Best Config: architecture = [50, 800, 500, 300, 2], lambda = 5e-07, decay = 0.0001, momentum = 0.0, actfn = relu, best_acc = 0.799999982119
Mean Time = 7.04892234008seconds, |Models| = 6, Total Time = 42.2935340405seconds
('Best Decay', 0.0001)

In [10]:

    

# (j)
print("\n\n## (j) SGD Momentum")
args['reg_coeffs'] = [0.0]
args['num_epoch'] = 50
args['sgd_decays'] = [best_decay] # TODO: get this from the best value of previous step
args['sgd_Nesterov'] = True
args['sgd_moms']= [0.99, 0.98, 0.95, 0.9, 0.85]
best, _ = testmodels(X_tr, Y_tr, X_te, Y_te, archs, **args)
best_mom = best[3]
print("Best moemntum", best_mom)


    

    





## (j) SGD Momentum
architecture=[50, 800, 500, 300, 2], lambda=0.0, decay=0.0001, momentum=0.99, actfn=relu: score=0.65000000596 | time=3.50205802917
architecture=[50, 800, 500, 300, 2], lambda=0.0, decay=0.0001, momentum=0.98, actfn=relu: score=0.65000000596 | time=3.60273885727
architecture=[50, 800, 500, 300, 2], lambda=0.0, decay=0.0001, momentum=0.95, actfn=relu: score=0.59999999404 | time=3.38674712181
architecture=[50, 800, 500, 300, 2], lambda=0.0, decay=0.0001, momentum=0.9, actfn=relu: score=0.75 | time=3.39339518547
architecture=[50, 800, 500, 300, 2], lambda=0.0, decay=0.0001, momentum=0.85, actfn=relu: score=0.550000011921 | time=3.41920590401
Best Config: architecture = [50, 800, 500, 300, 2], lambda = 0.0, decay = 0.0001, momentum = 0.9, actfn = relu, best_acc = 0.75
Mean Time = 3.46082901955seconds, |Models| = 5, Total Time = 17.3041450977seconds
('Best moemntum', 0.9)

In [11]:

    

# (k)
print("\n\n## (k) Combining all")
args['num_epoch'] = 100
args['sgd_lr'] = 1e-5
args['sgd_Nesterov'] = True
args['EStop'] = True

#TODO: Best values from previous steps
args['sgd_decays'] = [best_decay]
args['sgd_moms']= [best_mom]
args['reg_coeffs'] = [best_lambda_EStop]

testmodels(X_tr, Y_tr, X_te, Y_te, archs, **args)


    

    





## (k) Combining all
Epoch 00014: early stopping
architecture=[50, 800, 500, 300, 2], lambda=1e-07, decay=0.0001, momentum=0.9, actfn=relu: score=0.65000000596 | time=3.09857201576
Best Config: architecture = [50, 800, 500, 300, 2], lambda = 1e-07, decay = 0.0001, momentum = 0.9, actfn = relu, best_acc = 0.65000000596
Mean Time = 3.09857201576seconds, |Models| = 1, Total Time = 3.09857201576seconds






    
Out[11]:





([[50, 800, 500, 300, 2], 1e-07, 0.0001, 0.9, 'relu', 0.65000000596046448],
 array([ 3.09857202]))

In [12]:

    

# (l) Grid Search
print("\n\n## (j) Grid Search ")
archs = [[d_in, 50, d_out], [d_in, 500, d_out], [d_in, 500, 300, d_out],
         [d_in, 800, 500, 300, d_out], [d_in, 800, 800, 500, 300, d_out]]
args = {
    'actfn':'relu',
    'last_act':'softmax',
    'num_epoch': 100,
    'batch_size': 1000,
    'sgd_lr': 1e-5,
    'sgd_Nesterov': True,
    'sgd_moms': [0.99],
    'EStop': True,
    'verbose': False,
    'reg_coeffs': [1e-7, 5e-7, 1e-6, 5e-6, 1e-5],
    'sgd_decays': [1e-5, 5e-5, 1e-4],
}
testmodels(X_tr, Y_tr, X_te, Y_te, archs, **args)


    

    





## (j) Grid Search 
Epoch 00007: early stopping
architecture=[50, 50, 2], lambda=1e-07, decay=1e-05, momentum=0.99, actfn=relu: score=0.899999976158 | time=1.21030306816
Epoch 00007: early stopping
architecture=[50, 50, 2], lambda=1e-07, decay=5e-05, momentum=0.99, actfn=relu: score=0.449999988079 | time=1.2862830162
Epoch 00007: early stopping
architecture=[50, 50, 2], lambda=1e-07, decay=0.0001, momentum=0.99, actfn=relu: score=0.40000000596 | time=1.39065003395
Epoch 00007: early stopping
architecture=[50, 50, 2], lambda=5e-07, decay=1e-05, momentum=0.99, actfn=relu: score=0.25 | time=1.21554899216
Epoch 00007: early stopping
architecture=[50, 50, 2], lambda=5e-07, decay=5e-05, momentum=0.99, actfn=relu: score=0.25 | time=1.33358192444
Epoch 00007: early stopping
architecture=[50, 50, 2], lambda=5e-07, decay=0.0001, momentum=0.99, actfn=relu: score=0.300000011921 | time=1.34109997749
Epoch 00007: early stopping
architecture=[50, 50, 2], lambda=1e-06, decay=1e-05, momentum=0.99, actfn=relu: score=0.15000000596 | time=1.25166296959
Epoch 00007: early stopping
architecture=[50, 50, 2], lambda=1e-06, decay=5e-05, momentum=0.99, actfn=relu: score=0.449999988079 | time=1.30975294113
Epoch 00007: early stopping
architecture=[50, 50, 2], lambda=1e-06, decay=0.0001, momentum=0.99, actfn=relu: score=0.699999988079 | time=1.35929894447
Epoch 00012: early stopping
architecture=[50, 50, 2], lambda=5e-06, decay=1e-05, momentum=0.99, actfn=relu: score=0.600000023842 | time=1.31907081604
Epoch 00007: early stopping
architecture=[50, 50, 2], lambda=5e-06, decay=5e-05, momentum=0.99, actfn=relu: score=0.550000011921 | time=1.33358502388
Epoch 00007: early stopping
architecture=[50, 50, 2], lambda=5e-06, decay=0.0001, momentum=0.99, actfn=relu: score=0.699999988079 | time=1.47901797295
Epoch 00007: early stopping
architecture=[50, 50, 2], lambda=1e-05, decay=1e-05, momentum=0.99, actfn=relu: score=0.40000000596 | time=1.31222605705
Epoch 00009: early stopping
architecture=[50, 50, 2], lambda=1e-05, decay=5e-05, momentum=0.99, actfn=relu: score=0.649999976158 | time=1.47111296654
Epoch 00007: early stopping
architecture=[50, 50, 2], lambda=1e-05, decay=0.0001, momentum=0.99, actfn=relu: score=0.25 | time=1.27420687675
Epoch 00010: early stopping
architecture=[50, 500, 2], lambda=1e-07, decay=1e-05, momentum=0.99, actfn=relu: score=0.449999988079 | time=1.46441507339
Epoch 00007: early stopping
architecture=[50, 500, 2], lambda=1e-07, decay=5e-05, momentum=0.99, actfn=relu: score=0.34999999404 | time=1.4722058773
Epoch 00007: early stopping
architecture=[50, 500, 2], lambda=1e-07, decay=0.0001, momentum=0.99, actfn=relu: score=0.15000000596 | time=1.31370186806
Epoch 00007: early stopping
architecture=[50, 500, 2], lambda=5e-07, decay=1e-05, momentum=0.99, actfn=relu: score=0.300000011921 | time=1.3919467926
Epoch 00007: early stopping
architecture=[50, 500, 2], lambda=5e-07, decay=5e-05, momentum=0.99, actfn=relu: score=0.600000023842 | time=1.38250207901
Epoch 00007: early stopping
architecture=[50, 500, 2], lambda=5e-07, decay=0.0001, momentum=0.99, actfn=relu: score=0.600000023842 | time=1.37583708763
Epoch 00007: early stopping
architecture=[50, 500, 2], lambda=1e-06, decay=1e-05, momentum=0.99, actfn=relu: score=0.600000023842 | time=1.36326503754
Epoch 00007: early stopping
architecture=[50, 500, 2], lambda=1e-06, decay=5e-05, momentum=0.99, actfn=relu: score=0.449999988079 | time=1.24810910225
Epoch 00007: early stopping
architecture=[50, 500, 2], lambda=1e-06, decay=0.0001, momentum=0.99, actfn=relu: score=0.800000011921 | time=1.37515521049
Epoch 00007: early stopping
architecture=[50, 500, 2], lambda=5e-06, decay=1e-05, momentum=0.99, actfn=relu: score=0.300000011921 | time=1.38642597198
Epoch 00007: early stopping
architecture=[50, 500, 2], lambda=5e-06, decay=5e-05, momentum=0.99, actfn=relu: score=0.600000023842 | time=1.28251886368
Epoch 00007: early stopping
architecture=[50, 500, 2], lambda=5e-06, decay=0.0001, momentum=0.99, actfn=relu: score=0.449999988079 | time=1.35393500328
Epoch 00007: early stopping
architecture=[50, 500, 2], lambda=1e-05, decay=1e-05, momentum=0.99, actfn=relu: score=0.550000011921 | time=1.23606085777
Epoch 00007: early stopping
architecture=[50, 500, 2], lambda=1e-05, decay=5e-05, momentum=0.99, actfn=relu: score=0.699999988079 | time=1.32297897339
Epoch 00007: early stopping
architecture=[50, 500, 2], lambda=1e-05, decay=0.0001, momentum=0.99, actfn=relu: score=0.699999988079 | time=1.31705713272
Epoch 00007: early stopping
architecture=[50, 500, 300, 2], lambda=1e-07, decay=1e-05, momentum=0.99, actfn=relu: score=0.600000023842 | time=1.72202992439
Epoch 00007: early stopping
architecture=[50, 500, 300, 2], lambda=1e-07, decay=5e-05, momentum=0.99, actfn=relu: score=0.800000011921 | time=1.77702879906
Epoch 00007: early stopping
architecture=[50, 500, 300, 2], lambda=1e-07, decay=0.0001, momentum=0.99, actfn=relu: score=0.600000023842 | time=1.85370397568
Epoch 00007: early stopping
architecture=[50, 500, 300, 2], lambda=5e-07, decay=1e-05, momentum=0.99, actfn=relu: score=0.5 | time=1.79700708389
Epoch 00007: early stopping
architecture=[50, 500, 300, 2], lambda=5e-07, decay=5e-05, momentum=0.99, actfn=relu: score=0.34999999404 | time=1.84643912315
Epoch 00007: early stopping
architecture=[50, 500, 300, 2], lambda=5e-07, decay=0.0001, momentum=0.99, actfn=relu: score=0.5 | time=1.83488607407
Epoch 00007: early stopping
architecture=[50, 500, 300, 2], lambda=1e-06, decay=1e-05, momentum=0.99, actfn=relu: score=0.600000023842 | time=1.68571710587
Epoch 00007: early stopping
architecture=[50, 500, 300, 2], lambda=1e-06, decay=5e-05, momentum=0.99, actfn=relu: score=0.850000023842 | time=1.80838012695
Epoch 00007: early stopping
architecture=[50, 500, 300, 2], lambda=1e-06, decay=0.0001, momentum=0.99, actfn=relu: score=0.40000000596 | time=1.81895780563
Epoch 00007: early stopping
architecture=[50, 500, 300, 2], lambda=5e-06, decay=1e-05, momentum=0.99, actfn=relu: score=0.5 | time=1.83311510086
Epoch 00007: early stopping
architecture=[50, 500, 300, 2], lambda=5e-06, decay=5e-05, momentum=0.99, actfn=relu: score=0.5 | time=1.68818712234
Epoch 00007: early stopping
architecture=[50, 500, 300, 2], lambda=5e-06, decay=0.0001, momentum=0.99, actfn=relu: score=0.34999999404 | time=1.85359096527
Epoch 00007: early stopping
architecture=[50, 500, 300, 2], lambda=1e-05, decay=1e-05, momentum=0.99, actfn=relu: score=0.449999988079 | time=1.98514699936
Epoch 00007: early stopping
architecture=[50, 500, 300, 2], lambda=1e-05, decay=5e-05, momentum=0.99, actfn=relu: score=0.449999988079 | time=2.13911509514
Epoch 00007: early stopping
architecture=[50, 500, 300, 2], lambda=1e-05, decay=0.0001, momentum=0.99, actfn=relu: score=0.15000000596 | time=1.99159502983
Epoch 00008: early stopping
architecture=[50, 800, 500, 300, 2], lambda=1e-07, decay=1e-05, momentum=0.99, actfn=relu: score=0.449999988079 | time=2.97883582115
Epoch 00007: early stopping
architecture=[50, 800, 500, 300, 2], lambda=1e-07, decay=5e-05, momentum=0.99, actfn=relu: score=0.600000023842 | time=2.65910792351
Epoch 00007: early stopping
architecture=[50, 800, 500, 300, 2], lambda=1e-07, decay=0.0001, momentum=0.99, actfn=relu: score=0.40000000596 | time=2.57715702057
Epoch 00007: early stopping
architecture=[50, 800, 500, 300, 2], lambda=5e-07, decay=1e-05, momentum=0.99, actfn=relu: score=0.40000000596 | time=2.76571798325
Epoch 00007: early stopping
architecture=[50, 800, 500, 300, 2], lambda=5e-07, decay=5e-05, momentum=0.99, actfn=relu: score=0.600000023842 | time=2.67143392563
Epoch 00007: early stopping
architecture=[50, 800, 500, 300, 2], lambda=5e-07, decay=0.0001, momentum=0.99, actfn=relu: score=0.40000000596 | time=2.74462294579
Epoch 00007: early stopping
architecture=[50, 800, 500, 300, 2], lambda=1e-06, decay=1e-05, momentum=0.99, actfn=relu: score=0.449999988079 | time=2.81343007088
Epoch 00007: early stopping
architecture=[50, 800, 500, 300, 2], lambda=1e-06, decay=5e-05, momentum=0.99, actfn=relu: score=0.600000023842 | time=2.76092219353
Epoch 00007: early stopping
architecture=[50, 800, 500, 300, 2], lambda=1e-06, decay=0.0001, momentum=0.99, actfn=relu: score=0.649999976158 | time=2.72733187675
Epoch 00007: early stopping
architecture=[50, 800, 500, 300, 2], lambda=5e-06, decay=1e-05, momentum=0.99, actfn=relu: score=0.800000011921 | time=2.41824197769
Epoch 00007: early stopping
architecture=[50, 800, 500, 300, 2], lambda=5e-06, decay=5e-05, momentum=0.99, actfn=relu: score=0.600000023842 | time=2.54658317566
Epoch 00007: early stopping
architecture=[50, 800, 500, 300, 2], lambda=5e-06, decay=0.0001, momentum=0.99, actfn=relu: score=0.550000011921 | time=2.38805699348
Epoch 00007: early stopping
architecture=[50, 800, 500, 300, 2], lambda=1e-05, decay=1e-05, momentum=0.99, actfn=relu: score=0.600000023842 | time=2.25120592117
Epoch 00007: early stopping
architecture=[50, 800, 500, 300, 2], lambda=1e-05, decay=5e-05, momentum=0.99, actfn=relu: score=0.800000011921 | time=2.45753717422
Epoch 00007: early stopping
architecture=[50, 800, 500, 300, 2], lambda=1e-05, decay=0.0001, momentum=0.99, actfn=relu: score=0.34999999404 | time=2.56502890587
Epoch 00007: early stopping
architecture=[50, 800, 800, 500, 300, 2], lambda=1e-07, decay=1e-05, momentum=0.99, actfn=relu: score=0.699999988079 | time=3.2302839756
Epoch 00007: early stopping
architecture=[50, 800, 800, 500, 300, 2], lambda=1e-07, decay=5e-05, momentum=0.99, actfn=relu: score=0.300000011921 | time=3.14152598381
Epoch 00007: early stopping
architecture=[50, 800, 800, 500, 300, 2], lambda=1e-07, decay=0.0001, momentum=0.99, actfn=relu: score=0.40000000596 | time=3.25892901421
Epoch 00007: early stopping
architecture=[50, 800, 800, 500, 300, 2], lambda=5e-07, decay=1e-05, momentum=0.99, actfn=relu: score=0.600000023842 | time=3.30806398392
Epoch 00007: early stopping
architecture=[50, 800, 800, 500, 300, 2], lambda=5e-07, decay=5e-05, momentum=0.99, actfn=relu: score=0.649999976158 | time=3.17942905426
Epoch 00007: early stopping
architecture=[50, 800, 800, 500, 300, 2], lambda=5e-07, decay=0.0001, momentum=0.99, actfn=relu: score=0.40000000596 | time=3.21294498444
Epoch 00007: early stopping
architecture=[50, 800, 800, 500, 300, 2], lambda=1e-06, decay=1e-05, momentum=0.99, actfn=relu: score=0.40000000596 | time=3.48409605026
Epoch 00007: early stopping
architecture=[50, 800, 800, 500, 300, 2], lambda=1e-06, decay=5e-05, momentum=0.99, actfn=relu: score=0.40000000596 | time=3.06573796272
Epoch 00007: early stopping
architecture=[50, 800, 800, 500, 300, 2], lambda=1e-06, decay=0.0001, momentum=0.99, actfn=relu: score=0.40000000596 | time=3.17743301392
Epoch 00011: early stopping
architecture=[50, 800, 800, 500, 300, 2], lambda=5e-06, decay=1e-05, momentum=0.99, actfn=relu: score=0.75 | time=3.32091283798
Epoch 00007: early stopping
architecture=[50, 800, 800, 500, 300, 2], lambda=5e-06, decay=5e-05, momentum=0.99, actfn=relu: score=0.75 | time=3.04501795769
Epoch 00007: early stopping
architecture=[50, 800, 800, 500, 300, 2], lambda=5e-06, decay=0.0001, momentum=0.99, actfn=relu: score=0.550000011921 | time=3.07281899452
Epoch 00007: early stopping
architecture=[50, 800, 800, 500, 300, 2], lambda=1e-05, decay=1e-05, momentum=0.99, actfn=relu: score=0.699999988079 | time=3.09461283684
Epoch 00007: early stopping
architecture=[50, 800, 800, 500, 300, 2], lambda=1e-05, decay=5e-05, momentum=0.99, actfn=relu: score=0.649999976158 | time=3.05219483376
Epoch 00007: early stopping
architecture=[50, 800, 800, 500, 300, 2], lambda=1e-05, decay=0.0001, momentum=0.99, actfn=relu: score=0.40000000596 | time=3.06941986084
Best Config: architecture = [50, 50, 2], lambda = 1e-07, decay = 1e-05, momentum = 0.99, actfn = relu, best_acc = 0.899999976158
Mean Time = 2.0646273613seconds, |Models| = 75, Total Time = 154.847052097seconds






    
Out[12]:





([[50, 50, 2], 1e-07, 1e-05, 0.99, 'relu', 0.89999997615814209],
 array([ 1.21030307,  1.28628302,  1.39065003,  1.21554899,  1.33358192,
         1.34109998,  1.25166297,  1.30975294,  1.35929894,  1.31907082,
         1.33358502,  1.47901797,  1.31222606,  1.47111297,  1.27420688,
         1.46441507,  1.47220588,  1.31370187,  1.39194679,  1.38250208,
         1.37583709,  1.36326504,  1.2481091 ,  1.37515521,  1.38642597,
         1.28251886,  1.353935  ,  1.23606086,  1.32297897,  1.31705713,
         1.72202992,  1.7770288 ,  1.85370398,  1.79700708,  1.84643912,
         1.83488607,  1.68571711,  1.80838013,  1.81895781,  1.8331151 ,
         1.68818712,  1.85359097,  1.985147  ,  2.1391151 ,  1.99159503,
         2.97883582,  2.65910792,  2.57715702,  2.76571798,  2.67143393,
         2.74462295,  2.81343007,  2.76092219,  2.72733188,  2.41824198,
         2.54658318,  2.38805699,  2.25120592,  2.45753717,  2.56502891,
         3.23028398,  3.14152598,  3.25892901,  3.30806398,  3.17942905,
         3.21294498,  3.48409605,  3.06573796,  3.17743301,  3.32091284,
         3.04501796,  3.07281899,  3.09461284,  3.05219483,  3.06941986]))

In [ ]: