In [150]:

    

from bayes_opt import BayesianOptimization
from bayes_opt.observer import JSONLogger
from bayes_opt.event import Events
from bayes_opt.util import load_logs


    

In [151]:

    

import numpy as np
import sys
import json
import matplotlib.pyplot as plt
from matplotlib import gridspec
from matplotlib.backends.backend_pdf import PdfPages


    

In [152]:

    

sys.path.insert(0, "../python")
import BayesianOptimizationCustom as bo


    

In [153]:

    

optimizer_custum = bo.BayesianOptimizationCustom('../config/params_type.json', 
                                                 '../config/params_init_probe.json', 
                                                 None,
                                                 'output',
                                                 'output', 
                                                 10, 
                                                 '../config/params_type.json',
                                                 1)
params_range = optimizer_custum.ParamsRange()
params_range


    

    
Out[153]:





{'num_den_layers_pre': (0, 5),
 'num_units_den_layers_pre': (1, 60),
 'num_den_layers_post': (0, 20),
 'num_units_den_layers_post': (1, 60),
 'num_rnn_layers': (1, 10),
 'num_units_rnn_layer': (1, 200),
 'learning_rate_exp': (-4, -2)}

In [156]:

    

optimizer = BayesianOptimization(
    f=None,
    pbounds=params_range,
    random_state=1, verbose=1
)
# New optimizer is loaded with previously seen points
load_logs(optimizer, logs=["../python/09_12_2019_v22_opt.json"])


    

    
Out[156]:





<bayes_opt.bayesian_optimization.BayesianOptimization at 0x7f51dd742be0>

In [157]:

    

optimizer._gp.fit(optimizer._space.params, optimizer._space.target)


    

    
Out[157]:





GaussianProcessRegressor(alpha=1e-06, copy_X_train=True,
                         kernel=Matern(length_scale=1, nu=2.5),
                         n_restarts_optimizer=25, normalize_y=True,
                         optimizer='fmin_l_bfgs_b',
                         random_state=RandomState(MT19937) at 0x7F51DD2FC468)

In [158]:

    

optimizer._space.params.shape


    

    
Out[158]:





(64, 7)

In [159]:

    

optimizer._space.params.shape


    

    
Out[159]:





(64, 7)

In [160]:

    

optimizer._space.keys


    

    
Out[160]:





['learning_rate_exp',
 'num_den_layers_post',
 'num_den_layers_pre',
 'num_rnn_layers',
 'num_units_den_layers_post',
 'num_units_den_layers_pre',
 'num_units_rnn_layer']

In [161]:

    

# params = {'activation_dense_post' : 0, 'activation_dense_pre': 1, 'dropout_rate_den_layers_post' : 2,
#           'dropout_rate_den_layers_pre' : 3, 'dropout_rate_rnn': 4, 'learning_rate_exp': 5, 'num_den_layers_post': 6, 
#           'num_den_layers_pre': 7, 'num_rnn_layers': 8, 'num_units_den_layers_post': 9, 
#           'num_units_den_layers_pre': 10, 'num_units_rnn_layer': 11, 'optimizers': 12, 
#           'rnn_type': 13}

params = {'learning_rate_exp' : 0,
          'num_den_layers_post' : 1,
          'num_den_layers_pre': 2,
          'num_rnn_layers': 3,
          'num_units_den_layers_post': 4,
          'num_units_den_layers_pre': 5,
          'num_units_rnn_layer' : 6
         }


    

In [162]:

    

params_pos = {}
for pos in range(len(optimizer._space.target)):
    params_pos[optimizer._space.target[pos]] = pos


    

In [163]:

    

order_targets = optimizer._space.target
order_targets = -np.sort(-order_targets)
best_10_order_targets = order_targets[:10]
print(best_10_order_targets)


    

    




[0.95178926 0.9517079  0.9513554  0.95042223 0.95010322 0.9493503
 0.94929928 0.94924182 0.94920516 0.9491477 ]

In [164]:

    

optimizer._space.target.argmax()


    

    
Out[164]:





25

In [165]:

    

best_params = []
for point in best_10_order_targets:
    print(params_pos[point])
    best_params.append(optimizer._space.params[params_pos[point]])
#     max_point


    

    




25
2
62
38
63
0
41
14
51
12

In [166]:

    

# optimizer._space.params[1]


    

In [167]:

    

params_score = {}
# params = []
for n in range(10):
    params_score[n] = best_params[n]


    

In [169]:

    

# with open('10_best_params', 'w') as f:
#         f.write(json.dumps(best_params, indent=4))


    

In [170]:

    

max_point = optimizer._space.params[optimizer._space.target.argmax()]
max_point


    

    
Out[170]:





array([-3., 13.,  0.,  5., 15.,  1., 74.])

In [171]:

    

best_params[n].shape[0]


    

    
Out[171]:





7

In [172]:

    

def CreatePlots(max_point, var, pdf, n_bin, plot_title):
    var_points = np.linspace(params_range[var][0], params_range[var][1], 1000)#np.array([0, 1, 2, 3, 4, 5])
    x = np.zeros((var_points.shape[0], max_point.shape[0]))
    for n in range(x.shape[0]):
        x[n, :] = max_point[:]
        x[n, params[var]] = var_points[n]
#     print(x[:,0])
    mu, sigma = optimizer._gp.predict(x, return_std=True)
    max_idx = np.argmax(mu)
    print('{} max: mu={}, val={}'.format(var, mu[max_idx], x[max_idx, params[var]]))
#     print('mu:', mu)
#     print('sigma:', sigma)
#     print('2 * sigma:', 2 * sigma)
    fig = plt.figure(figsize=(7, 6))
    ax = plt.gca()
    ax.plot(var_points, mu, '--', color='k', label='Prediction')
    ax.fill(np.concatenate([var_points, var_points[::-1]]), 
        np.concatenate([mu - 2 * sigma, (mu + 2 * sigma)[::-1]]),
        alpha=.6, fc='c', ec='None', label='95% confidence interval', color='yellow')
    ax.fill(np.concatenate([var_points, var_points[::-1]]), 
        np.concatenate([mu - 1 * sigma, (mu + 1 * sigma)[::-1]]),
        alpha=.6, fc='c', ec='None', label='68% confidence interval', color='green')
    ax.set_ylabel('Purity')
    ax.set_xlabel(plot_title)
    plt.title(plot_title)
    ax.legend(loc=2, bbox_to_anchor=(1.01, 1), borderaxespad=0.)
    plt.draw()
    plt.show()
    pdf.savefig(fig, bbox_inches='tight')
    plt.close()


    

In [173]:

    

for n in range(1):
    with PdfPages('opt_plots_param_number_09_12_{}.pdf'.format(n)) as pdf:
        for key,values in params_range.items():
            CreatePlots(best_params[n], key, pdf, 'n_bin', key)


    

    




num_den_layers_pre max: mu=0.950905192121013, val=0.050050050050050046






    













    




num_units_den_layers_pre max: mu=0.9661925051609614, val=24.8008008008008






    













    




num_den_layers_post max: mu=0.9539185520335423, val=11.571571571571571






    













    




num_units_den_layers_post max: mu=0.9560377398794344, val=16.17817817817818






    













    




num_rnn_layers max: mu=0.9508919698740783, val=5.0






    













    




num_units_rnn_layer max: mu=0.9612410477840285, val=76.2972972972973






    













    




learning_rate_exp max: mu=0.9540837581306505, val=-3.3413413413413413