In [ ]:

    

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
#import seaborn as sns; sns.set()


    

In [ ]:

    

class Experiment:
    def __init__(self, path):
        self.frame = pd.read_csv(path, index_col=0)
        self.durations = self.frame.get('duration').values
        self.costs = self.frame.drop(columns=['seed', 'duration'])
        self.mean, self.stddev, self.lq, self.uq = get_best(self.costs.values)
        
def get_best(data):
    n = data.shape[0]
    bests = np.zeros_like(data)
    for row in range(data.shape[0]):
        for col in range(data.shape[1]):
            bests[row, col] = np.min(data[row,:col+1])
    mean = np.mean(bests, axis=0)
    stddev = np.sqrt(np.var(bests, axis=0))
    lq, uq = np.percentile(bests, [25, 75], axis=0)
    return mean, stddev, lq, uq


    

In [ ]:

    

random = Experiment('costs_random.csv')
turbo = Experiment('costs_turbo.csv')
turbo_fallback = Experiment('costs_turbo_with_fallback.csv')
gpyopt = Experiment('costs_gpyopt.csv')


    

In [ ]:

    

plt.figure(figsize=(10, 6))
plt.bar(0, turbo.durations.mean(axis=0), color='b', label='turbo')
plt.bar(1, turbo_fallback.durations.mean(axis=0), color='g', label='turbo with fallback')
plt.bar(2, gpyopt.durations.mean(axis=0), color='orange', label='GPyOpt')
plt.gca().set_xticks([])
plt.title('mean duration')
plt.legend()
plt.show()


    

In [ ]:

    

plt.figure(figsize=(16,10))
xs = range(30)
plt.plot(xs, random.mean, label='random')
plt.fill_between(xs, random.lq, random.uq, alpha=0.1)

plt.plot(xs, turbo.mean, label='turbo')
plt.fill_between(xs, turbo.lq, turbo.uq, alpha=0.1)

plt.plot(xs, turbo_fallback.mean, label='turbo with fallback')
plt.fill_between(xs, turbo_fallback.lq, turbo_fallback.uq, alpha=0.1)

plt.plot(xs, gpyopt.mean, label='GPyOpt')
plt.fill_between(xs, gpyopt.lq, gpyopt.uq, alpha=0.1)

plt.ylim((-3.5, -1.25))
plt.axvline(x=5, linestyle=':', color='k')
plt.legend()
plt.show()


    

In [ ]:

    

plt.figure(figsize=(16,10))
plt.plot(range(30), random.costs.mean(axis=0), label='random')
plt.plot(range(30), turbo.costs.mean(axis=0), label='turbo')
plt.plot(range(30), turbo_fallback.costs.mean(axis=0), label='turbo with fallback')
plt.plot(range(30), gpyopt.costs.mean(axis=0), label='GPyOpt')
plt.axvline(x=5, linestyle=':', color='k')
plt.legend()
plt.show()


    

In [ ]: