A Demonstration of Exploitation/Exploration Trade-off

Inspired by the following notebook:



In [ ]:

    
%load_ext autoreload
%autoreload 2
%matplotlib inline
%config InlineBackend.figure_format = 'svg'



In [ ]:

    
import numpy as np
import sklearn.gaussian_process as gp
import matplotlib.pyplot as plt
import seaborn as sns; sns.set() # prettify matplotlib



In [ ]:

    
# local modules
import turbo as tb
import turbo.modules as tm
import turbo.gui.jupyter as tg
import turbo.plotting as tp



In [ ]:

    
# Make deterministic
np.random.seed(42)



In [ ]:

    
# Effects every function in this notebook
desired_extremum = 'max'
surrogate = 'GPy'

Target Function



In [ ]:

    
'''
def f(x):
    sf = 1 if desired_extremum == 'max' else -1
    return sf * (np.exp(-(x - 2)**2) + np.exp(-(x - 6)**2/10) + 1/ (x**2 + 1))
xmin, xmax = -2, 10
xs = np.linspace(xmin, xmax, 500)
ys = f(xs)
'''

#f = lambda x: 100 * np.sin(x**2/5) * np.cos(x*1.5) + 100
f = lambda x: np.sin(x**2/5) * np.cos(x*1.5)
xmin, xmax = 0, 12
xs = np.linspace(xmin, xmax, num=200)
ys = f(xs)

def find_best():
    ''' the optimisers are finding solutions better than the 500 divisions can find'''
    xs = np.linspace(xmin, xmax, 10000)
    best_ys = f(xs)
    best_x = np.argmax(best_ys) if desired_extremum == 'max' else np.argmin(best_ys)
    return xs[best_x], best_ys[best_x]
best_x, best_y = find_best()


plt.figure(figsize=(12,4))
plt.plot(xs, ys)
plt.plot(best_x, best_y, 'ro')
plt.show()

Helper Functions

These functions set up the optimiser to act like this library



In [ ]:

    
def run_optimiser(acquisition_function):
    '''
    create an optimiser which is configured to be as close those used by the
    Bayesian optimisation library this notebook is copying, with the acquisition
    function and arguments for it passed as parameters
    '''
    bounds = [('x', xmin, xmax)]
    op = tb.Optimiser(f, desired_extremum, bounds, pre_phase_trials=4, settings_preset='default')
    if surrogate == 'GPy':
        op.surrogate = tm.GPySurrogate()
    elif surrogate == 'scikit':
        op.surrogate = tm.SciKitGPSurrogate(model_params=dict(
            kernel = 1.0 * gp.kernels.Matern(nu=2.5),
            normalize_y = True
        ), training_iterations=10)
    else:
        raise ValueError()
    print('using surrogate: {}'.format(surrogate))
    op.acquisition = acquisition_function

    rec = tb.Recorder(op)
    tg.OptimiserProgressBar(op, close_when_complete=True)
    np.random.seed(42) # make each pre_phase the same
    op.run(max_trials=20)
    
    tp.plot_trial_1D(rec, param='x', trial_num=-1, true_objective=f, fig=plt.figure(figsize=(12, 8)))
    tp.plot_error(rec, true_best=best_y, fig_ax=plt.subplots(figsize=(12, 3)))

Upper Confidence Bound: Prefer Exploitation

$\beta=0.1$



In [ ]:

    
run_optimiser(tm.UCB(beta=0.1))

Upper Confidence Bound: Prefer Exploration

$\beta=10.0$



In [ ]:

    
run_optimiser(tm.UCB(beta=10))

Expected Value (Not Useful) (Upper Confidence Bound - Pure Exploitation)

$$\beta=0$$

(only the mean/expected value $\mu$ is taken into account, the uncertainty ($\sigma$) is ignored)



In [ ]:

    
run_optimiser(tm.UCB(beta=0))

Upper Confidence Bound - Pure Exploration (Not Useful)

$$\beta=\infty$$

(only the uncertainty ($\sigma$) is taken into account, $\mu$ is ignored)



In [ ]:

    
run_optimiser(tm.UCB(beta=float('inf')))

Expected Improvement: Prefer Exploitation

$\xi=0.0001$



In [ ]:

    
run_optimiser(tm.EI(xi=1e-4))

Expected Improvement: Prefer Exploration

$\xi=0.1$



In [ ]:

    
run_optimiser(tm.EI(xi=0.1))

Probability of Improvement: Prefer Exploitation

$\xi=0.0001$



In [ ]:

    
run_optimiser(tm.PI(xi=1e-4))

Probability of Improvement: Prefer Exploration

$\xi=0.1$



In [ ]:

    
run_optimiser(tm.PI(xi=0.1))



In [ ]: