Please use Python 2. Python 3 got Cpickle problem



In [1]:

    
import pandas as pd
import tensorflow as tf
from bayes_opt import BayesianOptimization
from sklearn.cross_validation import train_test_split
import numpy as np
import cPickle









    



/usr/local/lib/python2.7/dist-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.
  "This module will be removed in 0.20.", DeprecationWarning)



In [2]:

    
def unpickle(file):
    with open(file, 'rb') as fo:
        dict = cPickle.load(fo)
    return dict

def reshape_image(img):
    img = img.reshape([3, 32, 32])
    return img.transpose([1, 2, 0])

unique_name = unpickle('/home/husein/space/cifar/cifar-10-batches-py/batches.meta')['label_names']
cifar10 = unpickle('/home/husein/space/cifar/cifar-10-batches-py/data_batch_1')



In [3]:

    
x_data = cifar10['data'][:300, :]
y_data = cifar10['labels'][:300]
onehot = np.zeros((x_data.shape[0], len(unique_name)))
for i in range(x_data.shape[0]):
    onehot[i, y_data[i]] = 1.0
    
x_train, x_test, y_train, y_test, y_train_label, y_test_label = train_test_split(x_data, onehot, y_data, test_size = 0.20)

Activation function:
0- for sigmoid
1- for tanh
2- for relu

Now the constants are:
1- batch size : 10
2- epoch: 20
3- adaptive gradient descent
4- softmax with cross entropy
5- 2 fully connected layers

So you can change anything you want



In [4]:

    
def neural_network(fully_conn_size, len_layer_conv, kernel_size, learning_rate, pooling_size, multiply,
                   dropout_rate, beta, activation, batch_normalization, batch_size = 10):
    
    tf.reset_default_graph()
    if activation == 0:
        activation = tf.nn.sigmoid
    elif activation == 1:
        activation = tf.nn.tanh
    else:
        activation = tf.nn.relu
    
    def conv_layer(x, conv, out_shape):
        w = tf.Variable(tf.truncated_normal([conv, conv, int(x.shape[3]), out_shape]))
        b = tf.Variable(tf.truncated_normal([out_shape], stddev = 0.01))
        return tf.nn.conv2d(x, w, [1, 1, 1, 1], padding = 'SAME') + b
    
    def fully_connected(x, out_shape):
        w = tf.Variable(tf.truncated_normal([int(x.shape[1]), out_shape]))
        b = tf.Variable(tf.truncated_normal([out_shape], stddev = 0.01))
        return tf.matmul(x, w) + b
    
    def pooling(x, k = 2, stride = 2):
        return tf.nn.max_pool(x, ksize = [1, k, k, 1], strides = [1, stride, stride, 1], padding = 'SAME')
        
    X = tf.placeholder(tf.float32, (None, 32, 32, 3))
    Y = tf.placeholder(tf.float32, (None, len(unique_name)))
    train = tf.placeholder(tf.bool)
    for i in range(len_layer_conv):
        if i == 0:
            conv = activation(conv_layer(X, kernel_size, int(np.around(int(X.shape[3]) * multiply))))
        else:
            conv = activation(conv_layer(conv, kernel_size, int(np.around(int(conv.shape[3]) * multiply))))
        conv = pooling(conv, k = pooling_size, stride = pooling_size)
        if batch_normalization:
            conv = tf.layers.batch_normalization(conv, training = train)
        conv = tf.nn.dropout(conv, dropout_rate)
    print(conv.shape)
    output_shape = int(conv.shape[1]) * int(conv.shape[2]) * int(conv.shape[3])
    conv = tf.reshape(conv, [-1, output_shape])
    for i in range(2):
        if i == 0:
            fc = activation(fully_connected(conv, fully_conn_size))
        else:
            fc = activation(fully_connected(fc, fully_conn_size))
        fc = tf.nn.dropout(fc, dropout_rate)
        if batch_normalization:
            fc = tf.layers.batch_normalization(fc, training = train)
    logits = fully_connected(fc, len(unique_name))
    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels = Y, logits = logits))
    cost += sum(beta * tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables())
    optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)
    correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(Y, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
    
    sess = tf.InteractiveSession()
    sess.run(tf.global_variables_initializer())
    COST, TEST_COST, ACC, TEST_ACC = [], [], [], []
    for i in range(20):
        train_acc, train_loss = 0, 0
        for n in range(0, (x_train.shape[0] // batch_size) * batch_size, batch_size):
            batch_x = np.zeros((batch_size, 32, 32, 3))
            for k in range(batch_size):
                batch_x[k, :, :, :] = reshape_image(x_train[n + k, :])
            _, loss = sess.run([optimizer, cost], feed_dict = {X: batch_x, Y: y_train[n: n + batch_size, :], train: True})
            train_acc += sess.run(accuracy, feed_dict = {X: batch_x, Y: y_train[n: n + batch_size, :], train: False})
            train_loss += loss
        batch_x = np.zeros((x_test.shape[0], 32, 32, 3))
        for k in range(x_test.shape[0]):
            batch_x[k, :, :, :] = reshape_image(x_test[k, :])
        results = sess.run([cost, accuracy], feed_dict = {X: batch_x, Y: y_test, train: False})
        TEST_COST.append(results[0])
        TEST_ACC.append(results[1])
        train_loss /= (x_train.shape[0] // batch_size)
        train_acc /= (x_train.shape[0] // batch_size)
        ACC.append(train_acc)
        COST.append(train_loss)
    COST = np.array(COST).mean()
    TEST_COST = np.array(TEST_COST).mean()
    ACC = np.array(ACC).mean()
    TEST_ACC = np.array(TEST_ACC).mean()
    return COST, TEST_COST, ACC, TEST_ACC



In [5]:

    
def generate_nn(fully_conn_size, len_layer_conv, kernel_size, learning_rate, pooling_size, multiply,
                dropout_rate, beta, activation, batch_normalization):
    global accbest
    param = {
        'fully_conn_size' : int(np.around(fully_conn_size)),
        'len_layer_conv' : int(np.around(len_layer_conv)),
        'kernel_size': int(np.around(kernel_size)),
        'learning_rate' : max(min(learning_rate, 1), 0.0001),
        'pooling_size': int(np.around(pooling_size)),
        'multiply': multiply,
        'dropout_rate' : max(min(dropout_rate, 0.99), 0),
        'beta' : max(min(beta, 0.5), 0.000001),
        'activation': int(np.around(activation)),
        'batch_normalization' : int(np.around(batch_normalization))
    }
    learning_cost, valid_cost, learning_acc, valid_acc = neural_network(**param)
    print("stop after 20 iteration with train cost %f, valid cost %f, train acc %f, valid acc %f" % (learning_cost, valid_cost, learning_acc, valid_acc))
    if (valid_acc > accbest):
        costbest = valid_acc
    return valid_acc



In [6]:

    
accbest = 0.0
NN_BAYESIAN = BayesianOptimization(generate_nn, 
                              {'fully_conn_size': (16, 128),
                               'len_layer_conv': (3, 5),
                               'kernel_size': (2, 7),
                               'learning_rate': (0.0001, 1),
                               'pooling_size': (2, 4),
                               'multiply': (1, 3),
                               'dropout_rate': (0.1, 0.99),
                               'beta': (0.000001, 0.49),
                               'activation': (0, 2),
                               'batch_normalization': (0, 1)
                              })
NN_BAYESIAN.maximize(init_points = 20, n_iter = 40, acq = 'ei', xi = 0.0)









    



Initialization
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 Step |   Time |      Value |   activation |   batch_normalization |      beta |   dropout_rate |   fully_conn_size |   kernel_size |   learning_rate |   len_layer_conv |   multiply |   pooling_size | 
(?, 1, 1, 100)
stop after 20 iteration with train cost 2008170.116490, valid cost 486246.000000, train acc 0.151042, valid acc 0.133333
    1 | 00m06s |    0.13333 |       1.4565 |                0.8903 |    0.2951 |         0.5427 |           61.9462 |        4.0407 |          0.1692 |           4.7475 |     2.0314 |         3.4064 | 
(?, 1, 1, 40)
stop after 20 iteration with train cost 359.077862, valid cost 41.487034, train acc 0.200417, valid acc 0.131667
    2 | 00m02s |    0.13167 |       1.0275 |                0.4410 |    0.1257 |         0.6587 |           59.6901 |        3.0467 |          0.6025 |           3.9315 |     1.9162 |         3.0427 | 
(?, 2, 2, 16)
stop after 20 iteration with train cost 330.979133, valid cost 9.589639, train acc 0.198333, valid acc 0.133333
    3 | 00m04s |    0.13333 |       0.4947 |                0.6139 |    0.3523 |         0.7974 |           94.9314 |        5.8448 |          0.3150 |           3.3750 |     1.7794 |         2.9960 | 
(?, 2, 2, 168)
stop after 20 iteration with train cost 571.771640, valid cost 4.475763, train acc 0.217292, valid acc 0.133333
    4 | 00m02s |    0.13333 |       1.1266 |                0.0823 |    0.3143 |         0.4237 |           30.2263 |        2.8229 |          0.4735 |           3.8008 |     2.7505 |         2.1838 | 
(?, 2, 2, 237)
stop after 20 iteration with train cost nan, valid cost nan, train acc 0.148125, valid acc 0.133333
    5 | 00m02s |    0.13333 |       1.5303 |                0.1720 |    0.0419 |         0.2041 |           58.8475 |        3.1729 |          0.3426 |           4.0076 |     2.9666 |         2.0881 | 
(?, 1, 1, 73)
stop after 20 iteration with train cost 13302.843301, valid cost 2.315546, train acc 0.244792, valid acc 0.133333
    6 | 00m05s |    0.13333 |       0.8586 |                0.8299 |    0.3653 |         0.3140 |          104.6627 |        5.9643 |          0.9526 |           3.7000 |     2.2140 |         3.4398 | 
(?, 4, 4, 56)
stop after 20 iteration with train cost nan, valid cost nan, train acc 0.083542, valid acc 0.050000
    7 | 00m02s |    0.05000 |       1.9712 |                0.3703 |    0.4468 |         0.7693 |           27.9524 |        3.8054 |          0.5529 |           3.0131 |     2.6585 |         2.1260 | 
(?, 2, 2, 7)
stop after 20 iteration with train cost 2070.000343, valid cost 11.033470, train acc 0.220000, valid acc 0.133333
    8 | 00m05s |    0.13333 |       1.2985 |                0.8392 |    0.2644 |         0.3082 |          125.7212 |        2.4825 |          0.5585 |           3.6307 |     1.1996 |         2.1053 | 
(?, 1, 1, 184)
stop after 20 iteration with train cost 821.436800, valid cost 62.800335, train acc 0.200417, valid acc 0.129167
    9 | 00m02s |    0.12917 |       1.1918 |                0.3009 |    0.2188 |         0.4814 |          100.3948 |        2.5854 |          0.3794 |           3.9951 |     2.8247 |         3.1268 | 
(?, 2, 2, 27)
stop after 20 iteration with train cost nan, valid cost nan, train acc 0.083333, valid acc 0.050000
   10 | 00m01s |    0.05000 |       1.6808 |                0.3121 |    0.3826 |         0.6200 |           99.0217 |        2.6936 |          0.8531 |           3.4801 |     2.0875 |         2.7903 | 
(?, 1, 1, 237)
stop after 20 iteration with train cost 101.862314, valid cost 20.170437, train acc 0.202083, valid acc 0.130000
   11 | 00m02s |    0.13000 |       0.0386 |                0.2792 |    0.1298 |         0.2041 |           56.0794 |        2.1811 |          0.4325 |           3.7931 |     2.9668 |         3.4967 | 
(?, 1, 1, 12)
stop after 20 iteration with train cost 32.727261, valid cost 3.304677, train acc 0.200625, valid acc 0.133333
   12 | 00m05s |    0.13333 |       0.0957 |                0.9127 |    0.3811 |         0.7214 |          100.4090 |        4.2859 |          0.2571 |           4.9440 |     1.3391 |         3.7319 | 
(?, 1, 1, 182)
stop after 20 iteration with train cost nan, valid cost nan, train acc 0.083542, valid acc 0.050000
   13 | 00m03s |    0.05000 |       1.6693 |                0.1889 |    0.1313 |         0.2157 |           54.9208 |        4.0497 |          0.3237 |           4.8071 |     2.2457 |         2.3954 | 
(?, 2, 2, 164)
stop after 20 iteration with train cost 111.949912, valid cost 84.082855, train acc 0.216042, valid acc 0.129167
   14 | 00m02s |    0.12917 |       0.2117 |                0.2318 |    0.0171 |         0.7501 |           96.0475 |        2.8243 |          0.6559 |           4.3425 |     2.7253 |         2.1887 | 
(?, 1, 1, 9)
stop after 20 iteration with train cost 7.330570, valid cost 2.319196, train acc 0.258125, valid acc 0.133333
   15 | 00m04s |    0.13333 |       1.8424 |                0.5583 |    0.4786 |         0.1465 |           55.4223 |        2.8651 |          0.9967 |           3.2202 |     1.4879 |         3.6654 | 
(?, 1, 1, 18)
stop after 20 iteration with train cost nan, valid cost nan, train acc 0.083750, valid acc 0.050000
   16 | 00m02s |    0.05000 |       1.6451 |                0.1834 |    0.4319 |         0.4315 |          113.7749 |        4.6080 |          0.6933 |           3.7359 |     1.5416 |         3.0782 | 
(?, 1, 1, 126)
stop after 20 iteration with train cost 198.105689, valid cost 4.173694, train acc 0.201458, valid acc 0.131667
   17 | 00m02s |    0.13167 |       1.2770 |                0.4776 |    0.4513 |         0.2476 |          100.0807 |        2.3012 |          0.3148 |           3.9662 |     2.5190 |         3.4889 | 
(?, 1, 1, 236)
stop after 20 iteration with train cost 566.766217, valid cost 15.120625, train acc 0.201250, valid acc 0.134167
   18 | 00m02s |    0.13417 |       1.1439 |                0.1005 |    0.3424 |         0.8912 |          114.8697 |        3.1598 |          0.3192 |           3.6584 |     2.9492 |         3.7208 | 
(?, 1, 1, 3)
stop after 20 iteration with train cost 10.717329, valid cost 2.350919, train acc 0.220417, valid acc 0.133333
   19 | 00m05s |    0.13333 |       0.0145 |                0.5464 |    0.1622 |         0.6554 |          100.6420 |        5.4176 |          0.8350 |           3.5967 |     1.1097 |         3.5063 | 
(?, 1, 1, 103)
stop after 20 iteration with train cost 149.477733, valid cost 3.264791, train acc 0.245833, valid acc 0.133333
   20 | 00m02s |    0.13333 |       0.4013 |                0.3964 |    0.3400 |         0.9426 |           22.3087 |        5.1583 |          0.3912 |           4.0846 |     2.4608 |         2.7389 | 
Bayesian Optimization
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 Step |   Time |      Value |   activation |   batch_normalization |      beta |   dropout_rate |   fully_conn_size |   kernel_size |   learning_rate |   len_layer_conv |   multiply |   pooling_size | 
(?, 1, 1, 3)
stop after 20 iteration with train cost 2.818659, valid cost 2.717047, train acc 0.168125, valid acc 0.125833
   21 | 01m13s |    0.12583 |       0.0000 |                0.0000 |    0.0000 |         0.1000 |           16.0000 |        2.0000 |          1.0000 |           5.0000 |     1.0000 |         4.0000 | 
(?, 1, 1, 81)
stop after 20 iteration with train cost 25.373671, valid cost 24.730690, train acc 0.106250, valid acc 0.110000
   22 | 00m35s |    0.11000 |       0.0000 |                0.0000 |    0.0000 |         0.1000 |           76.1813 |        2.0000 |          0.0001 |           3.0000 |     3.0000 |         4.0000 | 
(?, 1, 1, 81)
stop after 20 iteration with train cost 32.782606, valid cost 32.204044, train acc 0.101875, valid acc 0.097500
   23 | 00m35s |    0.09750 |       0.0000 |                0.0000 |    0.0000 |         0.1000 |          128.0000 |        7.0000 |          0.0001 |           3.0000 |     3.0000 |         4.0000 | 
(?, 1, 1, 81)
stop after 20 iteration with train cost 9.998409, valid cost 30.124105, train acc 0.103125, valid acc 0.090000
   24 | 00m39s |    0.09000 |       0.0000 |                1.0000 |    0.0000 |         0.1000 |           97.2206 |        7.0000 |          0.0001 |           3.0000 |     3.0000 |         4.0000 | 
(?, 1, 1, 81)
stop after 20 iteration with train cost 5719.770920, valid cost 5713.321777, train acc 0.106250, valid acc 0.088333
   25 | 00m40s |    0.08833 |       0.0000 |                0.0000 |    0.4900 |         0.1000 |          106.6973 |        2.0000 |          0.0001 |           3.0000 |     3.0000 |         4.0000 | 
(?, 1, 1, 3)
stop after 20 iteration with train cost 2026.791490, valid cost 2024.371338, train acc 0.104583, valid acc 0.101667
   26 | 00m39s |    0.10167 |       0.0000 |                0.0000 |    0.4900 |         0.1000 |           86.3271 |        7.0000 |          0.0001 |           5.0000 |     1.0000 |         2.0000 | 
(?, 1, 1, 729)
stop after 20 iteration with train cost nan, valid cost nan, train acc 0.183542, valid acc 0.133333
   27 | 00m40s |    0.13333 |       2.0000 |                0.0000 |    0.0000 |         0.9900 |          122.8743 |        2.0000 |          1.0000 |           5.0000 |     3.0000 |         4.0000 | 
(?, 1, 1, 729)
stop after 20 iteration with train cost 484.075631, valid cost 2.319522, train acc 0.261667, valid acc 0.133333
   28 | 00m44s |    0.13333 |       0.0000 |                1.0000 |    0.4900 |         0.1000 |           38.3927 |        2.0000 |          1.0000 |           5.0000 |     3.0000 |         4.0000 | 
(?, 1, 1, 729)
stop after 20 iteration with train cost 161934.436914, valid cost 161735.515625, train acc 0.098542, valid acc 0.090833
   29 | 00m40s |    0.09083 |       0.0000 |                0.0000 |    0.4900 |         0.1000 |           91.1139 |        2.0000 |          0.0001 |           5.0000 |     3.0000 |         4.0000 | 
(?, 4, 4, 81)
stop after 20 iteration with train cost 19011.979427, valid cost 3125141504.000000, train acc 0.101667, valid acc 0.095000
   30 | 00m43s |    0.09500 |       2.0000 |                1.0000 |    0.4900 |         0.1000 |           67.0943 |        2.0000 |          0.0001 |           3.0000 |     3.0000 |         2.0000 | 
(?, 1, 1, 81)
stop after 20 iteration with train cost 94.321815, valid cost 22.718958, train acc 0.191042, valid acc 0.118333
   31 | 00m42s |    0.11833 |       0.0000 |                0.0000 |    0.4900 |         0.1000 |           64.4711 |        7.0000 |          1.0000 |           3.0000 |     3.0000 |         4.0000 | 
(?, 1, 1, 729)
stop after 20 iteration with train cost 12.932306, valid cost 13.331874, train acc 0.095000, valid acc 0.091667
   32 | 00m42s |    0.09167 |       0.0000 |                0.0000 |    0.0000 |         0.1000 |           26.9175 |        2.0000 |          0.0001 |           5.0000 |     3.0000 |         4.0000 | 
(?, 1, 1, 729)
stop after 20 iteration with train cost 1810420.614583, valid cost 1808203.375000, train acc 0.100833, valid acc 0.110000
   33 | 01m07s |    0.11000 |       0.0000 |                0.0000 |    0.4900 |         0.1000 |           16.0000 |        7.0000 |          0.0001 |           5.0000 |     3.0000 |         2.0000 | 
(?, 1, 1, 3)
stop after 20 iteration with train cost 4.182123, valid cost 3.660193, train acc 0.163333, valid acc 0.126667
   34 | 00m44s |    0.12667 |       0.0000 |                0.0000 |    0.0000 |         0.1000 |           32.3153 |        7.0000 |          1.0000 |           5.0000 |     1.0000 |         2.0000 | 
(?, 4, 4, 3)
stop after 20 iteration with train cost 4.835538, valid cost 2.605304, train acc 0.274792, valid acc 0.133333
   35 | 00m43s |    0.13333 |       0.0000 |                0.0000 |    0.4900 |         0.9900 |           34.5035 |        2.0000 |          1.0000 |           3.0000 |     1.0000 |         2.0000 | 
(?, 4, 4, 81)
stop after 20 iteration with train cost 33747.258602, valid cost 33724.734375, train acc 0.107917, valid acc 0.098333
   36 | 00m47s |    0.09833 |       0.0000 |                1.0000 |    0.4900 |         0.1000 |          121.3749 |        2.0000 |          0.0001 |           3.0000 |     3.0000 |         2.0000 | 
(?, 1, 1, 729)
stop after 20 iteration with train cost 455.433505, valid cost 11.401892, train acc 0.181667, valid acc 0.111667






    



/usr/local/lib/python2.7/dist-packages/sklearn/gaussian_process/gpr.py:427: UserWarning: fmin_l_bfgs_b terminated abnormally with the  state: {'warnflag': 2, 'task': 'ABNORMAL_TERMINATION_IN_LNSRCH', 'grad': array([ -2.71350418e-05]), 'nit': 4, 'funcalls': 47}
  " state: %s" % convergence_dict)






    



   37 | 00m46s |    0.11167 |       0.0000 |                0.0000 |    0.4900 |         0.1000 |           33.0480 |        2.0000 |          1.0000 |           5.0000 |     3.0000 |         2.0000 | 
(?, 1, 1, 81)
stop after 20 iteration with train cost nan, valid cost nan, train acc 0.096250, valid acc 0.066667
   38 | 00m45s |    0.06667 |       2.0000 |                0.0000 |    0.0000 |         0.1000 |           39.2425 |        2.0000 |          0.0001 |           3.0000 |     3.0000 |         4.0000 | 
(?, 1, 1, 3)
stop after 20 iteration with train cost 8.769759, valid cost 5.812512, train acc 0.078958, valid acc 0.115833
   39 | 00m46s |    0.11583 |       0.0000 |                1.0000 |    0.0000 |         0.9900 |           32.0688 |        3.5489 |          0.0001 |           3.0000 |     1.0000 |         4.0000 | 
(?, 1, 1, 3)
stop after 20 iteration with train cost 3.545781, valid cost 3.373584, train acc 0.082500, valid acc 0.115000
   40 | 00m44s |    0.11500 |       0.0000 |                0.0000 |    0.0000 |         0.9900 |           20.7268 |        4.5839 |          0.0001 |           5.0000 |     1.0000 |         2.0000 | 
(?, 1, 1, 729)
stop after 20 iteration with train cost 163574.322917, valid cost 163374.781250, train acc 0.090625, valid acc 0.091667
   41 | 00m46s |    0.09167 |       0.0000 |                0.0000 |    0.4900 |         0.1000 |          100.7270 |        2.0000 |          0.0001 |           5.0000 |     3.0000 |         4.0000 | 
(?, 4, 4, 3)
stop after 20 iteration with train cost 12.494074, valid cost 5.014549, train acc 0.105208, valid acc 0.139167
   42 | 00m46s |    0.13917 |       0.0000 |                1.0000 |    0.0000 |         0.9900 |           91.8885 |        3.8067 |          0.0001 |           3.0000 |     1.0000 |         2.0000 | 
(?, 1, 1, 81)
stop after 20 iteration with train cost 14.831696, valid cost 4481962.000000, train acc 0.084167, valid acc 0.050833
   43 | 00m46s |    0.05083 |       2.0000 |                1.0000 |    0.0000 |         0.9900 |          128.0000 |        2.0000 |          0.0001 |           3.0000 |     3.0000 |         4.0000 | 
(?, 1, 1, 81)
stop after 20 iteration with train cost 8.730399, valid cost 4077280512.000000, train acc 0.094583, valid acc 0.104167
   44 | 00m46s |    0.10417 |       2.0000 |                1.0000 |    0.0000 |         0.1000 |          103.2247 |        2.0837 |          0.0001 |           3.0000 |     3.0000 |         4.0000 | 
(?, 4, 4, 81)
stop after 20 iteration with train cost 33514.741565, valid cost 33492.203125, train acc 0.101667, valid acc 0.095833
   45 | 00m43s |    0.09583 |       0.0000 |                1.0000 |    0.4900 |         0.1000 |          101.3569 |        4.2432 |          0.0001 |           3.0240 |     3.0000 |         2.0000 | 
(?, 1, 1, 729)
stop after 20 iteration with train cost 14.724192, valid cost 20010471424.000000, train acc 0.106875, valid acc 0.164167
   46 | 01m12s |    0.16417 |       2.0000 |                1.0000 |    0.0000 |         0.9900 |           75.2335 |        7.0000 |          0.0001 |           5.0000 |     3.0000 |         4.0000 | 
(?, 1, 1, 3)
stop after 20 iteration with train cost 8.570062, valid cost 7.071836, train acc 0.219792, valid acc 0.121667
   47 | 00m44s |    0.12167 |       0.0000 |                0.0000 |    0.0000 |         0.9900 |          128.0000 |        2.0000 |          1.0000 |           5.0000 |     1.0000 |         2.0000 | 
(?, 1, 1, 81)
stop after 20 iteration with train cost 8768.209446, valid cost 2.319522, train acc 0.262292, valid acc 0.133333
   48 | 00m47s |    0.13333 |       0.0000 |                1.0000 |    0.4900 |         0.9900 |           42.8165 |        7.0000 |          1.0000 |           3.0000 |     3.0000 |         4.0000 | 
(?, 1, 1, 3)
stop after 20 iteration with train cost 15.033110, valid cost 3.558356, train acc 0.260833, valid acc 0.116667
   49 | 00m45s |    0.11667 |       0.0000 |                0.0000 |    0.4900 |         0.9900 |          122.0387 |        7.0000 |          1.0000 |           3.0000 |     1.0000 |         4.0000 | 
(?, 1, 1, 729)
stop after 20 iteration with train cost 16.263203, valid cost 8.534091, train acc 0.091250, valid acc 0.068333
   50 | 01m06s |    0.06833 |       0.0000 |                1.0000 |    0.0000 |         0.9900 |           92.1810 |        5.6116 |          0.0001 |           5.0000 |     3.0000 |         2.0000 | 
(?, 1, 1, 3)
stop after 20 iteration with train cost 1606.707641, valid cost 1605.977905, train acc 0.095208, valid acc 0.072500
   51 | 00m46s |    0.07250 |       0.0000 |                0.0000 |    0.4900 |         0.1000 |           76.8088 |        7.0000 |          0.0001 |           3.0000 |     1.0000 |         4.0000 | 
(?, 4, 4, 81)
stop after 20 iteration with train cost 2.460649, valid cost 2.379506, train acc 0.203958, valid acc 0.135833






    



/usr/local/lib/python2.7/dist-packages/sklearn/gaussian_process/gpr.py:427: UserWarning: fmin_l_bfgs_b terminated abnormally with the  state: {'warnflag': 2, 'task': 'ABNORMAL_TERMINATION_IN_LNSRCH', 'grad': array([ -3.24209129e-05]), 'nit': 3, 'funcalls': 55}
  " state: %s" % convergence_dict)






    



   52 | 00m48s |    0.13583 |       0.0000 |                1.0000 |    0.0000 |         0.9900 |           16.0000 |        2.0000 |          1.0000 |           3.0000 |     3.0000 |         2.0000 | 
(?, 1, 1, 3)
stop after 20 iteration with train cost 3.341574, valid cost 29347958784.000000, train acc 0.139792, valid acc 0.127500






    



/usr/local/lib/python2.7/dist-packages/sklearn/gaussian_process/gpr.py:427: UserWarning: fmin_l_bfgs_b terminated abnormally with the  state: {'warnflag': 2, 'task': 'ABNORMAL_TERMINATION_IN_LNSRCH', 'grad': array([ -3.15959728e-05]), 'nit': 4, 'funcalls': 53}
  " state: %s" % convergence_dict)






    



   53 | 00m52s |    0.12750 |       2.0000 |                1.0000 |    0.0000 |         0.9900 |           78.5702 |        2.0000 |          1.0000 |           5.0000 |     1.0000 |         4.0000 | 
(?, 1, 1, 3)
stop after 20 iteration with train cost 16.973149, valid cost 2.319522, train acc 0.262292, valid acc 0.133333
   54 | 00m51s |    0.13333 |       2.0000 |                1.0000 |    0.4900 |         0.9900 |          128.0000 |        7.0000 |          1.0000 |           5.0000 |     1.0000 |         2.0000 | 
(?, 1, 1, 81)
stop after 20 iteration with train cost 2.323403, valid cost 2552851456.000000, train acc 0.119583, valid acc 0.123333
   55 | 00m50s |    0.12333 |       2.0000 |                1.0000 |    0.0000 |         0.9900 |           59.7197 |        2.0000 |          1.0000 |           3.0000 |     3.0000 |         4.0000 | 
(?, 1, 1, 3)
stop after 20 iteration with train cost 3.903047, valid cost 2.319522, train acc 0.261875, valid acc 0.133333
   56 | 00m51s |    0.13333 |       0.0000 |                1.0000 |    0.4900 |         0.1000 |           46.0005 |        2.0000 |          1.0000 |           3.0000 |     1.0000 |         4.0000 | 
(?, 1, 1, 3)
stop after 20 iteration with train cost nan, valid cost nan, train acc 0.183542, valid acc 0.133333
   57 | 00m50s |    0.13333 |       2.0000 |                0.0000 |    0.0000 |         0.1000 |           70.8739 |        7.0000 |          1.0000 |           5.0000 |     1.0000 |         2.0000 | 
(?, 1, 1, 81)
stop after 20 iteration with train cost 2.474757, valid cost 2.377540, train acc 0.223958, valid acc 0.131667
   58 | 00m51s |    0.13167 |       0.0000 |                0.0000 |    0.0000 |         0.9900 |           18.5245 |        7.0000 |          1.0000 |           3.0000 |     3.0000 |         4.0000 | 
(?, 1, 1, 3)
stop after 20 iteration with train cost 5.571828, valid cost 5.417197, train acc 0.102500, valid acc 0.111667
   59 | 00m51s |    0.11167 |       0.0000 |                0.0000 |    0.0000 |         0.9900 |          105.8432 |        7.0000 |          0.0001 |           3.0000 |     1.0000 |         4.0000 | 
(?, 4, 4, 81)
stop after 20 iteration with train cost 30901.746834, valid cost 30873.255859, train acc 0.103333, valid acc 0.101667
   60 | 00m55s |    0.10167 |       0.0000 |                1.0000 |    0.4900 |         0.1000 |           34.7228 |        7.0000 |          0.0001 |           3.0000 |     3.0000 |         2.0000 |



In [7]:

    
print('Maximum NN accuracy value: %f' % NN_BAYESIAN.res['max']['max_val'])
print('Best NN parameters: %s' % NN_BAYESIAN.res['max']['max_params'])









    



Maximum NN accuracy value: 0.164167
Best NN parameters: {'fully_conn_size': 75.233542680827611, 'len_layer_conv': 5.0, 'activation': 2.0, 'dropout_rate': 0.98999999999999999, 'batch_normalization': 1.0, 'beta': 9.9999999999999995e-07, 'pooling_size': 4.0, 'multiply': 3.0, 'learning_rate': 0.0001, 'kernel_size': 7.0}



In [ ]: