notebook.community

Edit and run



In [1]:

    
import pandas as pd
import tensorflow as tf
from bayes_opt import BayesianOptimization
from sklearn.cross_validation import train_test_split
from sklearn.preprocessing import LabelEncoder
import numpy as np









    



/usr/local/lib/python3.5/dist-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.
  "This module will be removed in 0.20.", DeprecationWarning)

You can install Bayesian Optimization by,

pip install bayesian-optimization



In [2]:

    
df = pd.read_csv('Iris.csv')
df.head()









    Out[2]:







  
    
      
      Id
      SepalLengthCm
      SepalWidthCm
      PetalLengthCm
      PetalWidthCm
      Species
    
  
  
    
      0
      1
      5.1
      3.5
      1.4
      0.2
      Iris-setosa
    
    
      1
      2
      4.9
      3.0
      1.4
      0.2
      Iris-setosa
    
    
      2
      3
      4.7
      3.2
      1.3
      0.2
      Iris-setosa
    
    
      3
      4
      4.6
      3.1
      1.5
      0.2
      Iris-setosa
    
    
      4
      5
      5.0
      3.6
      1.4
      0.2
      Iris-setosa



In [3]:

    
x_data = df.iloc[:, :-1].values.astype(np.float32)
y_datalabel = df.iloc[:, -1]
y_data = LabelEncoder().fit_transform(df.iloc[:, -1])

onehot = np.zeros((y_data.shape[0], np.unique(y_data).shape[0]))
for i in range(y_data.shape[0]):
    onehot[i, y_data[i]] = 1.0
    
x_train, x_test, y_train, y_test, y_train_label, y_test_label = train_test_split(x_data, onehot, y_data, test_size = 0.2)

Activation function:
0- for sigmoid
1- for tanh
2- for relu

Now the constants are:
1- batch size : 16
2- epoch: 100
3- gradient descent
4- softmax with cross entropy

So you can change anything you want



In [4]:

    
def neural_network(num_hidden, size_layer, learning_rate, dropout_rate, beta, activation, batch_size = 16):
    
    def activate(activation, first_layer, second_layer, bias):
        if activation == 0:
            activation = tf.nn.sigmoid
        elif activation == 1:
            activation = tf.nn.tanh
        else:
            activation = tf.nn.relu
        layer = activation(tf.matmul(first_layer, second_layer) + bias)
        return tf.nn.dropout(layer, dropout_rate)

    tf.reset_default_graph()
    X = tf.placeholder(tf.float32, (None, x_data.shape[1]))
    Y = tf.placeholder(tf.float32, (None, onehot.shape[1]))
    input_layer = tf.Variable(tf.random_normal([x_data.shape[1], size_layer]))
    biased_layer = tf.Variable(tf.random_normal([size_layer], stddev = 0.1))
    output_layer = tf.Variable(tf.random_normal([size_layer, onehot.shape[1]]))
    biased_output = tf.Variable(tf.random_normal([onehot.shape[1]], stddev = 0.1))
    layers, biased = [], []
    for i in range(num_hidden - 1):
        layers.append(tf.Variable(tf.random_normal([size_layer, size_layer])))
        biased.append(tf.Variable(tf.random_normal([size_layer])))
    first_l = activate(activation, X, input_layer, biased_layer)
    next_l = activate(activation, first_l, layers[0], biased[0])
    for i in range(1, num_hidden - 1):
        next_l = activate(activation, next_l, layers[i], biased[i])
    last_l = tf.matmul(next_l, output_layer) + biased_output
    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = last_l, labels = Y))
    regularizers = tf.nn.l2_loss(input_layer) + sum(map(lambda x: tf.nn.l2_loss(x), layers)) + tf.nn.l2_loss(output_layer)
    cost = cost + beta * regularizers
    optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)
    correct_prediction = tf.equal(tf.argmax(last_l, 1), tf.argmax(Y, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
    
    sess = tf.InteractiveSession()
    sess.run(tf.global_variables_initializer())
    COST, TEST_COST, ACC, TEST_ACC = [], [], [], []
    for i in range(100):
        train_acc, train_loss = 0, 0
        for n in range(0, (x_train.shape[0] // batch_size) * batch_size, batch_size):
            _, loss = sess.run([optimizer, cost], feed_dict = {X: x_train[n: n + batch_size, :], Y: y_train[n: n + batch_size, :]})
            train_acc += sess.run(accuracy, feed_dict = {X: x_train[n: n + batch_size, :], Y: y_train[n: n + batch_size, :]})
            train_loss += loss
        TEST_COST.append(sess.run(cost, feed_dict = {X: x_test, Y: y_test}))
        TEST_ACC.append(sess.run(accuracy, feed_dict = {X: x_test, Y: y_test}))
        train_loss /= (x_train.shape[0] // batch_size)
        train_acc /= (x_train.shape[0] // batch_size)
        ACC.append(train_acc)
        COST.append(train_loss)
    COST = np.array(COST).mean()
    TEST_COST = np.array(TEST_COST).mean()
    ACC = np.array(ACC).mean()
    TEST_ACC = np.array(TEST_ACC).mean()
    return COST, TEST_COST, ACC, TEST_ACC



In [5]:

    
def generate_nn(num_hidden, size_layer, learning_rate, dropout_rate, beta, activation):
    global accbest
    param = {
        'num_hidden' : int(np.around(num_hidden)),
        'size_layer' : int(np.around(size_layer)),
        'learning_rate' : max(min(learning_rate, 1), 0.0001),
        'dropout_rate' : max(min(dropout_rate, 0.99), 0),
        'beta' : max(min(beta, 0.5), 0.000001),
        'activation': int(np.around(activation))
    }
    print("\nSearch parameters %s" % (param), file = log_file)
    log_file.flush()
    learning_cost, valid_cost, learning_acc, valid_acc = neural_network(**param)
    print("stop after 200 iteration with train cost %f, valid cost %f, train acc %f, valid acc %f" % (learning_cost, valid_cost, learning_acc, valid_acc))
    if (valid_acc > accbest):
        costbest = valid_acc
    return valid_acc

hidden layers (2, 20)
layer size (32, 1024)
learning rate (0.0001, 1)
dropout rate (0.1, 0.99)
beta (0.000001, 0.49)
activation (0, 2)

You can set your own minimum and maximum boundaries, just change the value



In [6]:

    
log_file = open('nn-bayesian.log', 'a')
accbest = 0.0
NN_BAYESIAN = BayesianOptimization(generate_nn, 
                              {'num_hidden': (2, 20),
                               'size_layer': (32, 1024),
                               'learning_rate': (0.0001, 1),
                               'dropout_rate': (0.1, 0.99),
                               'beta': (0.000001, 0.49),
                               'activation': (0, 2)
                              })
NN_BAYESIAN.maximize(init_points = 30, n_iter = 50, acq = 'ei', xi = 0.0)









    



Initialization
-------------------------------------------------------------------------------------------------------------------------
 Step |   Time |      Value |   activation |      beta |   dropout_rate |   learning_rate |   num_hidden |   size_layer | 
stop after 200 iteration with train cost 9.328038, valid cost 6.830622, train acc 0.477589, valid acc 0.344333
    1 | 00m00s |    0.34433 |       0.1557 |    0.0741 |         0.5189 |          0.6100 |       2.2780 |     109.6324 | 
stop after 200 iteration with train cost nan, valid cost nan, train acc 0.339286, valid acc 0.333333
    2 | 00m06s |    0.33333 |       1.5493 |    0.2110 |         0.9648 |          0.6746 |       8.4695 |     406.6496 | 
stop after 200 iteration with train cost 353764719888.678528, valid cost 323302490112.000000, train acc 0.407589, valid acc 0.338667
    3 | 00m19s |    0.33867 |       1.2247 |    0.0866 |         0.4345 |          0.1704 |      10.1839 |     715.3899 | 
stop after 200 iteration with train cost 643.186734, valid cost 562.714478, train acc 0.452679, valid acc 0.336333
    4 | 00m06s |    0.33633 |       0.3547 |    0.0210 |         0.3734 |          0.8197 |       3.5302 |     673.8076 | 
stop after 200 iteration with train cost 2221.470260, valid cost 1842.391724, train acc 0.445357, valid acc 0.357000
    5 | 00m05s |    0.35700 |       0.2858 |    0.1263 |         0.2995 |          0.1810 |       5.0905 |     522.6034 | 
stop after 200 iteration with train cost 6961.388417, valid cost 6149.663574, train acc 0.493750, valid acc 0.413667
    6 | 00m05s |    0.41367 |       0.6033 |    0.1827 |         0.5848 |          0.0870 |       4.1379 |     703.5684 | 
stop after 200 iteration with train cost 349.111370, valid cost 345.697021, train acc 0.419464, valid acc 0.323000
    7 | 00m01s |    0.32300 |       1.1583 |    0.0067 |         0.6424 |          0.3208 |       7.8891 |      39.8767 | 
stop after 200 iteration with train cost 23666985409133.800781, valid cost 9528878825472.000000, train acc 0.378393, valid acc 0.337667
    8 | 00m17s |    0.33767 |       0.8956 |    0.2294 |         0.9459 |          0.5383 |      12.8620 |     612.1061 | 
stop after 200 iteration with train cost 8036.225374, valid cost 2387.566650, train acc 0.467321, valid acc 0.366000
    9 | 00m11s |    0.36600 |       0.1703 |    0.4454 |         0.7967 |          0.3245 |      17.3349 |     368.9528 | 
stop after 200 iteration with train cost nan, valid cost nan, train acc 0.339286, valid acc 0.333333
   10 | 00m11s |    0.33333 |       1.5674 |    0.3451 |         0.6319 |          0.0612 |       8.7600 |     578.6664 | 
stop after 200 iteration with train cost nan, valid cost nan, train acc 0.339286, valid acc 0.333333
   11 | 00m13s |    0.33333 |       1.8639 |    0.2297 |         0.3017 |          0.2796 |       6.5510 |     742.1305 | 
stop after 200 iteration with train cost nan, valid cost nan, train acc 0.339286, valid acc 0.333333
   12 | 00m20s |    0.33333 |       1.9607 |    0.2509 |         0.2129 |          0.7851 |      18.0818 |     552.7377 | 
stop after 200 iteration with train cost 912546814612081.500000, valid cost 410255880093696.000000, train acc 0.360536, valid acc 0.330667
   13 | 00m10s |    0.33067 |       0.5600 |    0.1149 |         0.8199 |          0.9516 |      15.6145 |     367.4106 | 
stop after 200 iteration with train cost 2202.113165, valid cost 2200.553955, train acc 0.337589, valid acc 0.318333
   14 | 00m01s |    0.31833 |       0.3462 |    0.1185 |         0.7723 |          0.0008 |      14.4203 |      54.6657 | 
stop after 200 iteration with train cost 13782066.450716, valid cost 13049213.000000, train acc 0.401875, valid acc 0.343000
   15 | 00m12s |    0.34300 |       0.3512 |    0.3970 |         0.1535 |          0.0305 |      12.8067 |     521.1275 | 
stop after 200 iteration with train cost nan, valid cost nan, train acc 0.340089, valid acc 0.333333
   16 | 00m01s |    0.33333 |       1.5210 |    0.4836 |         0.6073 |          0.0967 |       2.4645 |     447.8342 | 
stop after 200 iteration with train cost nan, valid cost nan, train acc 0.339286, valid acc 0.333333
   17 | 00m04s |    0.33333 |       1.7349 |    0.1968 |         0.6056 |          0.6338 |       8.0812 |     350.4353 | 
stop after 200 iteration with train cost 1947060954815615.750000, valid cost 866631119011840.000000, train acc 0.360357, valid acc 0.327333
   18 | 00m31s |    0.32733 |       1.3231 |    0.4887 |         0.7716 |          0.2276 |      14.4337 |     805.2767 | 
stop after 200 iteration with train cost 122298.837863, valid cost 92612.437500, train acc 0.412054, valid acc 0.339000
   19 | 00m06s |    0.33900 |       0.6309 |    0.1844 |         0.1738 |          0.2662 |       3.5663 |     736.8381 | 
stop after 200 iteration with train cost nan, valid cost nan, train acc 0.339286, valid acc 0.333333
   20 | 00m08s |    0.33333 |       1.7122 |    0.4791 |         0.9071 |          0.0297 |      15.6884 |     331.4725 | 
stop after 200 iteration with train cost 42225580.909496, valid cost 33537108.000000, train acc 0.452857, valid acc 0.363333
   21 | 00m39s |    0.36333 |       0.2802 |    0.0669 |         0.8116 |          0.5377 |      19.1366 |     781.4634 | 
stop after 200 iteration with train cost 22334.224615, valid cost 22224.960938, train acc 0.344107, valid acc 0.327333
   22 | 00m15s |    0.32733 |       0.4945 |    0.0198 |         0.7045 |          0.0314 |      14.0202 |     503.2816 | 
stop after 200 iteration with train cost 3477.443581, valid cost 1217.328125, train acc 0.459821, valid acc 0.366000
   23 | 00m15s |    0.36600 |       0.4481 |    0.2197 |         0.5347 |          0.5192 |       7.1655 |     844.7446 | 
stop after 200 iteration with train cost 1934.195407, valid cost 524.202576, train acc 0.460804, valid acc 0.366000
   24 | 00m08s |    0.36600 |       0.4756 |    0.3660 |         0.8134 |          0.3746 |      18.5520 |     304.2112 | 
stop after 200 iteration with train cost 278460277637984096.000000, valid cost 241075363273244672.000000, train acc 0.384196, valid acc 0.340000
   25 | 00m30s |    0.34000 |       0.5748 |    0.1577 |         0.9098 |          0.2183 |      14.1637 |     821.2710 | 
stop after 200 iteration with train cost nan, valid cost nan, train acc 0.339286, valid acc 0.333333
   26 | 00m20s |    0.33333 |       1.8351 |    0.4040 |         0.9243 |          0.2144 |      17.6275 |     598.4533 | 
stop after 200 iteration with train cost nan, valid cost nan, train acc 0.339286, valid acc 0.333333
   27 | 00m12s |    0.33333 |       1.7924 |    0.1764 |         0.9181 |          0.3998 |       6.1969 |     849.8700 | 
stop after 200 iteration with train cost nan, valid cost nan, train acc 0.339286, valid acc 0.333333
   28 | 00m04s |    0.33333 |       1.7188 |    0.3961 |         0.7138 |          0.5098 |      16.2234 |     188.7646 | 
stop after 200 iteration with train cost 6165122670396218.000000, valid cost 1843995621720064.000000, train acc 0.368750, valid acc 0.332667
   29 | 00m15s |    0.33267 |       0.5595 |    0.2736 |         0.6995 |          0.5692 |      15.1249 |     545.9633 | 
stop after 200 iteration with train cost nan, valid cost nan, train acc 0.339286, valid acc 0.333333
   30 | 00m07s |    0.33333 |       1.5275 |    0.1780 |         0.3050 |          0.4046 |      13.4876 |     352.6378 | 
Bayesian Optimization
-------------------------------------------------------------------------------------------------------------------------
 Step |   Time |      Value |   activation |      beta |   dropout_rate |   learning_rate |   num_hidden |   size_layer | 
stop after 200 iteration with train cost 1266.822980, valid cost 747.736511, train acc 0.406786, valid acc 0.348667
   31 | 00m36s |    0.34867 |       0.0000 |    0.4900 |         0.1000 |          1.0000 |       2.0000 |    1024.0000 | 
stop after 200 iteration with train cost 15492.492482, valid cost 15486.300781, train acc 0.334107, valid acc 0.330333






    



/usr/local/lib/python3.5/dist-packages/sklearn/gaussian_process/gpr.py:457: UserWarning: fmin_l_bfgs_b terminated abnormally with the  state: {'warnflag': 2, 'nit': 6, 'funcalls': 57, 'task': b'ABNORMAL_TERMINATION_IN_LNSRCH', 'grad': array([ -5.80776418e-05])}
  " state: %s" % convergence_dict)






    



   32 | 00m15s |    0.33033 |       0.0000 |    0.4900 |         0.1000 |          0.0001 |       2.0000 |     251.8187 | 
stop after 200 iteration with train cost 295507838599718848.000000, valid cost 473933199441920.000000, train acc 0.409196, valid acc 0.334667
   33 | 01m13s |    0.33467 |       0.0000 |    0.4900 |         0.1000 |          1.0000 |      20.0000 |     964.5148 | 
stop after 200 iteration with train cost 516649316.413125, valid cost 528222688.000000, train acc 0.335268, valid acc 0.334667
   34 | 00m28s |    0.33467 |       0.0000 |    0.4900 |         0.1000 |          0.0001 |      20.0000 |     394.5861 | 
stop after 200 iteration with train cost 546875561072.333618, valid cost 550831259648.000000, train acc 0.330357, valid acc 0.318000






    



/usr/local/lib/python3.5/dist-packages/sklearn/gaussian_process/gpr.py:457: UserWarning: fmin_l_bfgs_b terminated abnormally with the  state: {'warnflag': 2, 'nit': 5, 'funcalls': 55, 'task': b'ABNORMAL_TERMINATION_IN_LNSRCH', 'grad': array([ -5.02832199e-05])}
  " state: %s" % convergence_dict)
/usr/local/lib/python3.5/dist-packages/sklearn/gaussian_process/gpr.py:457: UserWarning: fmin_l_bfgs_b terminated abnormally with the  state: {'warnflag': 2, 'nit': 4, 'funcalls': 50, 'task': b'ABNORMAL_TERMINATION_IN_LNSRCH', 'grad': array([ -7.68838636e-05])}
  " state: %s" % convergence_dict)






    



   35 | 01m18s |    0.31800 |       0.0000 |    0.4900 |         0.1000 |          0.0001 |      20.0000 |    1024.0000 | 
stop after 200 iteration with train cost 87570.081942, valid cost 87536.796875, train acc 0.367232, valid acc 0.313667






    



/usr/local/lib/python3.5/dist-packages/sklearn/gaussian_process/gpr.py:457: UserWarning: fmin_l_bfgs_b terminated abnormally with the  state: {'warnflag': 2, 'nit': 6, 'funcalls': 53, 'task': b'ABNORMAL_TERMINATION_IN_LNSRCH', 'grad': array([  2.04261366e-05])}
  " state: %s" % convergence_dict)






    



   36 | 00m18s |    0.31367 |       0.0000 |    0.4900 |         0.9900 |          0.0001 |      20.0000 |     139.0137 | 
stop after 200 iteration with train cost 144748.068772, valid cost 144693.312500, train acc 0.320268, valid acc 0.328000






    



/usr/local/lib/python3.5/dist-packages/sklearn/gaussian_process/gpr.py:457: UserWarning: fmin_l_bfgs_b terminated abnormally with the  state: {'warnflag': 2, 'nit': 3, 'funcalls': 52, 'task': b'ABNORMAL_TERMINATION_IN_LNSRCH', 'grad': array([ -2.12913283e-05])}
  " state: %s" % convergence_dict)






    



   37 | 00m15s |    0.32800 |       0.0000 |    0.4900 |         0.1000 |          0.0001 |       2.0000 |     778.1937 | 
stop after 200 iteration with train cost 117422.896362, valid cost 117377.101562, train acc 0.332321, valid acc 0.340333
   38 | 00m22s |    0.34033 |       0.0000 |    0.4900 |         0.1000 |          0.0001 |       2.0000 |     698.8917 | 
stop after 200 iteration with train cost nan, valid cost nan, train acc 0.339286, valid acc 0.333333






    



/usr/local/lib/python3.5/dist-packages/sklearn/gaussian_process/gpr.py:457: UserWarning: fmin_l_bfgs_b terminated abnormally with the  state: {'warnflag': 2, 'nit': 3, 'funcalls': 51, 'task': b'ABNORMAL_TERMINATION_IN_LNSRCH', 'grad': array([ -3.36248811e-05])}
  " state: %s" % convergence_dict)






    



   39 | 00m45s |    0.33333 |       2.0000 |    0.0000 |         0.9900 |          1.0000 |      20.0000 |     655.3416 | 
stop after 200 iteration with train cost 129.469567, valid cost 134.233414, train acc 0.387143, valid acc 0.331667
   40 | 00m22s |    0.33167 |       0.0000 |    0.0000 |         0.9900 |          1.0000 |       2.0000 |     924.9424 | 
stop after 200 iteration with train cost 2.360393, valid cost 2.157934, train acc 0.701964, valid acc 0.699667
   41 | 00m06s |    0.69967 |       0.0000 |    0.0000 |         0.9900 |          0.0001 |       2.0000 |     985.9104 | 
stop after 200 iteration with train cost 4227.413826, valid cost 5810.387695, train acc 0.793482, valid acc 0.731667






    



/usr/local/lib/python3.5/dist-packages/sklearn/gaussian_process/gpr.py:457: UserWarning: fmin_l_bfgs_b terminated abnormally with the  state: {'warnflag': 2, 'nit': 4, 'funcalls': 50, 'task': b'ABNORMAL_TERMINATION_IN_LNSRCH', 'grad': array([  1.25955252e-05])}
  " state: %s" % convergence_dict)






    



   42 | 00m05s |    0.73167 |       2.0000 |    0.0000 |         0.9900 |          0.0001 |       2.0000 |     979.2264 | 
stop after 200 iteration with train cost 114.748760, valid cost 109.977303, train acc 0.385714, valid acc 0.331667
   43 | 00m14s |    0.33167 |       0.0000 |    0.0000 |         0.9900 |          1.0000 |       2.0000 |     974.0893 | 
stop after 200 iteration with train cost nan, valid cost nan, train acc 0.339286, valid acc 0.333333
   44 | 00m38s |    0.33333 |       2.0000 |    0.0411 |         0.5573 |          0.0001 |       8.9461 |     989.1162 | 
stop after 200 iteration with train cost nan, valid cost nan, train acc 0.339286, valid acc 0.333333
   45 | 01m00s |    0.33333 |       2.0000 |    0.4900 |         0.1000 |          0.0001 |      20.0000 |     898.4754 | 
stop after 200 iteration with train cost 89389.414922, valid cost 91162.671875, train acc 0.782143, valid acc 0.726333
   46 | 00m17s |    0.72633 |       2.0000 |    0.1775 |         0.9900 |          0.0001 |       2.0000 |     983.3027 | 
stop after 200 iteration with train cost nan, valid cost nan, train acc 0.339286, valid acc 0.333333
   47 | 00m18s |    0.33333 |       2.0000 |    0.0000 |         0.9900 |          1.0000 |      20.0000 |     226.3106 | 
stop after 200 iteration with train cost nan, valid cost nan, train acc 0.339107, valid acc 0.333333
   48 | 00m13s |    0.33333 |       2.0000 |    0.0000 |         0.1000 |          1.0000 |       2.0000 |     160.3019 | 
stop after 200 iteration with train cost nan, valid cost nan, train acc 0.339286, valid acc 0.333333
   49 | 00m20s |    0.33333 |       2.0000 |    0.0000 |         0.9900 |          0.0001 |      20.0000 |     270.6803 | 
stop after 200 iteration with train cost nan, valid cost nan, train acc 0.339286, valid acc 0.333333






    



/usr/local/lib/python3.5/dist-packages/sklearn/gaussian_process/gpr.py:457: UserWarning: fmin_l_bfgs_b terminated abnormally with the  state: {'warnflag': 2, 'nit': 7, 'funcalls': 58, 'task': b'ABNORMAL_TERMINATION_IN_LNSRCH', 'grad': array([ -1.14290191e-05])}
  " state: %s" % convergence_dict)






    



   50 | 01m04s |    0.33333 |       2.0000 |    0.0000 |         0.1000 |          0.0001 |      20.0000 |     932.1613 | 
stop after 200 iteration with train cost 1790989561259.659424, valid cost 2872419072.000000, train acc 0.417321, valid acc 0.340000
   51 | 00m30s |    0.34000 |       0.0000 |    0.4900 |         0.1000 |          1.0000 |      20.0000 |     466.5532 | 
stop after 200 iteration with train cost nan, valid cost nan, train acc 0.339286, valid acc 0.333333
   52 | 00m16s |    0.33333 |       2.0000 |    0.4900 |         0.1000 |          1.0000 |      20.0000 |      90.3217 | 
stop after 200 iteration with train cost 23599.251629, valid cost 23225.775391, train acc 0.339196, valid acc 0.327667
   53 | 00m12s |    0.32767 |       2.0000 |    0.4900 |         0.1000 |          0.0001 |       2.0000 |     291.1782 | 
stop after 200 iteration with train cost 10849.599879, valid cost 10846.084961, train acc 0.330982, valid acc 0.333000
   54 | 00m12s |    0.33300 |       0.0000 |    0.4900 |         0.1000 |          0.0001 |       2.0000 |     209.7478 | 
stop after 200 iteration with train cost 836698.001875, valid cost 836369.187500, train acc 0.310536, valid acc 0.357333
   55 | 00m25s |    0.35733 |       0.0000 |    0.4900 |         0.9900 |          0.0001 |      20.0000 |     430.9495 | 
stop after 200 iteration with train cost 2610.505313, valid cost 3743.947266, train acc 0.775536, valid acc 0.704333
   56 | 00m12s |    0.70433 |       2.0000 |    0.0000 |         0.9900 |          0.0001 |       2.0000 |     640.0483 | 
stop after 200 iteration with train cost 525.780525, valid cost 279.063110, train acc 0.413393, valid acc 0.354667
   57 | 00m13s |    0.35467 |       0.0000 |    0.4900 |         0.1000 |          1.0000 |       2.0000 |     648.9397 | 
stop after 200 iteration with train cost 2511.024381, valid cost 4094.970703, train acc 0.773929, valid acc 0.707000
   58 | 00m12s |    0.70700 |       2.0000 |    0.0000 |         0.9900 |          0.0001 |       2.0000 |     631.0850 | 
stop after 200 iteration with train cost nan, valid cost nan, train acc 0.339286, valid acc 0.333333
   59 | 00m26s |    0.33333 |       2.0000 |    0.0000 |         0.9900 |          0.0001 |       9.8101 |     635.0291 | 
stop after 200 iteration with train cost 2103.426130, valid cost 3523.851807, train acc 0.775179, valid acc 0.704000
   60 | 00m12s |    0.70400 |       2.0000 |    0.0000 |         0.9900 |          0.0001 |       2.0000 |     604.4642 | 
stop after 200 iteration with train cost nan, valid cost nan, train acc 0.339286, valid acc 0.333333
   61 | 00m42s |    0.33333 |       2.0000 |    0.0000 |         0.9900 |          0.0001 |      20.0000 |     688.6037 | 
stop after 200 iteration with train cost 11778684163032600297865216.000000, valid cost 4127465904021539815161856.000000, train acc 0.455625, valid acc 0.302000
   62 | 00m10s |    0.30200 |       2.0000 |    0.4900 |         0.9900 |          1.0000 |       2.0000 |      75.1183 | 
stop after 200 iteration with train cost 2363.404740, valid cost 3338.792725, train acc 0.768661, valid acc 0.700333
   63 | 00m12s |    0.70033 |       2.0000 |    0.0000 |         0.9900 |          0.0001 |       2.0000 |     615.6993 | 
stop after 200 iteration with train cost 54.370196, valid cost 55.933582, train acc 0.340625, valid acc 0.336667
   64 | 00m14s |    0.33667 |       0.0000 |    0.0000 |         0.1000 |          0.0001 |       2.0000 |     886.5317 | 
stop after 200 iteration with train cost 57608.024777, valid cost 58273.511719, train acc 0.772411, valid acc 0.708667
   65 | 00m11s |    0.70867 |       2.0000 |    0.4900 |         0.9900 |          0.0001 |       2.0000 |     480.8877 | 
stop after 200 iteration with train cost 306.521649, valid cost 140.108978, train acc 0.420982, valid acc 0.335667
   66 | 00m11s |    0.33567 |       0.0000 |    0.4900 |         0.1000 |          1.0000 |       2.0000 |     488.7328 | 
stop after 200 iteration with train cost 1847.159662, valid cost 2675.446777, train acc 0.755357, valid acc 0.698667
   67 | 00m12s |    0.69867 |       2.0000 |    0.0000 |         0.9900 |          0.0001 |       2.0000 |     472.8665 | 
stop after 200 iteration with train cost nan, valid cost nan, train acc 0.339375, valid acc 0.333333
   68 | 00m19s |    0.33333 |       2.0000 |    0.4900 |         0.9900 |          0.0001 |       6.2140 |     477.5421 | 
stop after 200 iteration with train cost 4421836022181.369141, valid cost 4428154798080.000000, train acc 0.368929, valid acc 0.322667
   69 | 01m01s |    0.32267 |       0.0000 |    0.0000 |         0.1000 |          1.0000 |      20.0000 |     866.0937 | 
stop after 200 iteration with train cost nan, valid cost nan, train acc 0.339375, valid acc 0.333333
   70 | 00m16s |    0.33333 |       2.0000 |    0.4900 |         0.1000 |          1.0000 |      20.0000 |      32.0000 | 
stop after 200 iteration with train cost 23832.248702, valid cost 27512.738281, train acc 0.339643, valid acc 0.328333
   71 | 00m14s |    0.32833 |       2.0000 |    0.0000 |         0.1000 |          0.0001 |       2.0000 |     980.9223 | 
stop after 200 iteration with train cost 243752.976563, valid cost 245191.812500, train acc 0.792946, valid acc 0.721000
   72 | 00m16s |    0.72100 |       2.0000 |    0.4900 |         0.9900 |          0.0001 |       2.0000 |    1002.2185 | 
stop after 200 iteration with train cost 44.434253, valid cost 46.155499, train acc 0.360446, valid acc 0.343000
   73 | 00m13s |    0.34300 |       0.0000 |    0.0000 |         0.1000 |          1.0000 |       2.0000 |     316.6161 | 
stop after 200 iteration with train cost 238715.202254, valid cost 239937.000000, train acc 0.784821, valid acc 0.725667
   74 | 00m16s |    0.72567 |       2.0000 |    0.4900 |         0.9900 |          0.0001 |       2.0000 |     992.2111 | 
stop after 200 iteration with train cost nan, valid cost nan, train acc 0.339286, valid acc 0.333333
   75 | 00m13s |    0.33333 |       2.0000 |    0.0000 |         0.1000 |          1.0000 |       2.0000 |     133.5501 | 
stop after 200 iteration with train cost 35653.372400, valid cost 35638.906250, train acc 0.327589, valid acc 0.332667
   76 | 00m14s |    0.33267 |       0.0000 |    0.4900 |         0.1000 |          0.0001 |       2.0000 |     384.2866 | 
stop after 200 iteration with train cost 22809745.857813, valid cost 36581.242188, train acc 0.452857, valid acc 0.361667
   77 | 00m50s |    0.36167 |       0.0000 |    0.4900 |         0.9900 |          1.0000 |      20.0000 |     759.4352 | 
stop after 200 iteration with train cost nan, valid cost nan, train acc 0.339286, valid acc 0.333333
   78 | 00m16s |    0.33333 |       2.0000 |    0.0000 |         0.1000 |          1.0000 |      20.0000 |     115.6470 | 
stop after 200 iteration with train cost 123782.659542, valid cost 123733.789062, train acc 0.322054, valid acc 0.341000
   79 | 00m18s |    0.34100 |       0.0000 |    0.4900 |         0.9900 |          0.0001 |      20.0000 |     165.8287 | 
stop after 200 iteration with train cost nan, valid cost nan, train acc 0.339911, valid acc 0.333333
   80 | 00m15s |    0.33333 |       2.0000 |    0.0000 |         0.9900 |          1.0000 |       2.0000 |     557.3990 |



In [7]:

    
print('Maximum NN accuracy value: %f' % NN_BAYESIAN.res['max']['max_val'])
print('Best NN parameters: ', NN_BAYESIAN.res['max']['max_params'])









    



Maximum NN accuracy value: 0.731667
Best NN parameters:  {'dropout_rate': 0.98999999999999999, 'beta': 9.9999999999999995e-07, 'learning_rate': 0.0001, 'size_layer': 979.22638102861038, 'activation': 2.0, 'num_hidden': 2.0}

So that means, best optimized parameters are:

dropout rate: 0.98999999999999999
beta: 9.9999999999999995e-07
learning rate: 0.0001
size layer: 979 wide
activation function: relu
hidden layers: 2

	Id	SepalLengthCm	SepalWidthCm	PetalLengthCm	PetalWidthCm	Species
0	1	5.1	3.5	1.4	0.2	Iris-setosa
1	2	4.9	3.0	1.4	0.2	Iris-setosa
2	3	4.7	3.2	1.3	0.2	Iris-setosa
3	4	4.6	3.1	1.5	0.2	Iris-setosa
4	5	5.0	3.6	1.4	0.2	Iris-setosa