In [1]:
import itertools
import os.path
import sys
sys.path.append('../plume')
from neatplots.colors import create_distinct_colors, MplLCHColor, SequentialLCPalette
from neatplots.tools import SharedAxesGrid
from neatplots.visualize import plot_colors
from matplotlib.gridspec import GridSpec, SubplotSpec
import tables
%pylab inline
import latexstyle
latexstyle.setup()
In [2]:
kappas = ['0.1', '0.5', '0.75', '1', '1.25', '1.5', '2']
gammas = ['0', '-1e-9', '-1e-8', '-1e-7', '-1e-6', '-1e-5', '-1e-4', '-1e-3', '-1e-2']
In [3]:
gray = (0.3, 0.3, 0.3)
seq_palette = SequentialLCPalette('seq', 210, (60, 100), (90, 0))
var2d_palette = np.ones((len(kappas), len(gammas), 3))
base_colors = create_distinct_colors(len(gammas), 10, 360, 30, 90)
for i, gamma in enumerate(gammas):
base = base_colors[i]
sp = SequentialLCPalette('', base.hue, (95, base.luminance), (90, 90))
var2d_palette[:, i, :] = sp.discretize(len(kappas))
def colormat_legend(ax, colors):
for i in xrange(colors.shape[1]):
plot_colors(colors[:, i, :], pos=(-0.5, i - 0.5), axes=ax)
ax.set_aspect(0.25)
ax.set_xlim(-0.5, len(kappas) - 0.5)
ax.set_ylim(-0.5, len(gammas) - 0.5)
ax.set_xticks(range(len(kappas)))
ax.set_yticks(range(len(gammas)))
ax.set_xticks(np.arange(-0.5, len(kappas) - 0.5), minor=True)
ax.set_yticks(np.arange(-0.5, len(gammas) - 0.5), minor=True)
ax.set_xticklabels([float(kappa) for kappa in kappas])
ax.set_yticklabels([float(gamma) for gamma in gammas])
ax.tick_params(which='major', length=0)
ax.tick_params(which='minor', length=rcParams['xtick.major.size'])
ax.set_xlabel(r'$\kappa$', labelpad=0)
ax.set_ylabel(r'$\gamma$', labelpad=0)
In [4]:
def to_paramstr(behavior, mse_scaling, kappa, gamma):
if behavior == 'GO':
return "_{}_gamma={}".format(behavior, gamma)
else:
return "_{}_scaling={!r} kappa={} gamma={}".format(behavior, mse_scaling, kappa, gamma)
def iter_trial_filenames(basename, start=0):
i = start
while True:
filename = os.path.join('..', 'Data', '{}.{}.h5'.format(basename, i))
if not os.path.isfile(filename):
if i == start:
warnings.warn("No datafiles for '{}'.".format(basename))
return
yield filename
i += 1
def extend_data(a, size):
cur_size = a.size
a = np.resize(a, size)
a[cur_size:] = a[cur_size - 1]
return a
def cached(fn):
cache = {}
def fn_with_cache(*args):
try:
return cache[args]
except:
val = fn(*args)
cache[args] = fn
return val
return fn_with_cache
In [5]:
err_names = {'rmse': 'RMISE', 'wrmse': 'WRMISE', 'rewards': 'QRSim reward'}
@cached
def get_final_errors(experiment, behavior, mse_scaling, kappas, gammas, error_type):
mean = np.ones((len(kappas), len(gammas)))
std = np.zeros((len(kappas), len(gammas)))
for (i, kappa), (j, gamma) in itertools.product(enumerate(kappas), enumerate(gammas)):
paramstr = to_paramstr(behavior, mse_scaling, kappa, gamma)
values = []
for filename in iter_trial_filenames(os.path.join(experiment, paramstr)):
with tables.open_file(filename) as fileh:
err_trace = fileh.get_node(fileh.root, error_type)
values.append(err_trace[-1] / err_trace[0])
if len(values) > 0:
mean[i, j] = np.mean(values)
std[i, j] = np.std(values)
return mean, std
@cached
def get_final_error_unnorm(experiment, behavior, mse_scaling, kappa, gamma, error_type):
if experiment == 'SingleSourceGaussian':
err_cor_path = '../Data/g-nf-ss-sv.errcor.h5'
elif experiment == 'SingleSourceGaussianDispersionNoiseless':
err_cor_path = '../Data/d-nf-ss-sv.errcor.h5'
elif experiment == 'MultiSourceGaussianDispersionNoiseless':
err_cor_path = '../Data/d-nf-ms-sv.errcor.h5'
with tables.open_file(err_cor_path) as fileh:
err_cor = fileh.root.errcor.read()
paramstr = to_paramstr(behavior, mse_scaling, kappa, gamma)
values = []
for i, filename in enumerate(iter_trial_filenames(os.path.join(experiment, paramstr))):
with tables.open_file(filename) as fileh:
err_trace = fileh.get_node(fileh.root, error_type)
values.append(err_trace[-1] * err_cor[i])
return np.mean(values), np.std(values)
def plot_final_errors(ax, experiment, behavior, mse_scaling, kappas, gammas, error_type):
values, unused = get_final_errors(experiment, behavior, mse_scaling, tuple(kappas), tuple(gammas), error_type)
f_gammas = [float(g) for g in gammas]
f_kappas = [float(k) for k in kappas]
idx_min = np.unravel_index(values.argmin(), values.shape)
csf = ax.contourf(
f_kappas, f_gammas, values.T, levels=np.arange(0.0, 1.1, 0.1), cmap=seq_palette)
cs = ax.contour(
f_kappas, f_gammas, values.T, levels=np.arange(0.0, 1.1, 0.1), colors=(0.3, 0.3, 0.3))
#ax.clabel(cs, fmt='%1.1f')
if behavior == 'GO':
k = 1.0
else:
k = f_kappas[idx_min[0]]
ax.scatter(k, f_gammas[idx_min[1]], marker='x', s=30, c='k')
auto = ''
if mse_scaling == 'auto':
auto = 'auto-scaled '
ax.set_title('{err}, {auto}{behavior}'.format(
err=err_names[error_type], behavior=behavior, auto=auto))
ax.set_yscale('symlog', linthreshy=1e-10)
ax.set_ylim(min(f_gammas), max(f_gammas))
ax.set_xlim(min(f_kappas), max(f_kappas))
return csf
def print_min_error(experiment, behavior, scaling, kappas, gammas, error_type):
mean, std = get_final_errors(experiment, behavior, scaling, tuple(kappas), tuple(gammas), error_type)
idx = np.unravel_index(np.argmin(mean), mean.shape)
gamma = gammas[idx[1]]
kappa = kappas[idx[0]]
umean, ustd = get_final_error_unnorm(experiment, behavior, scaling, kappa, gamma, error_type)
if scaling == 'auto':
algo = 'auto-scaled ' + behavior
else:
algo = behavior
print r'{algo} & {kappa:.2f} & \num{{{gamma}}} & {umean:.2f} & {ustd:.2f} & {mean:.2f} & {std:.2f} \\'.format(
algo=algo, kappa=float(kappa), gamma=float(gamma), umean=umean * 1e9, ustd=ustd * 1e9, mean=mean[idx], std=std[idx])
def create_final_error_matrix(experiment, behaviors, error_types):
fig = plt.figure(figsize=(5.35, 6.2))
legend_height = 0.025
grid = GridSpec(
2, 1, height_ratios=(1.0 - legend_height, legend_height),
hspace=0.09, bottom=0.05, left=0.1, top=0.95, right=0.9725)
class ParamSearchPlot(SharedAxesGrid):
def _create_axes(self, subplot_spec, sharex, sharey):
return fig.add_subplot(subplot_spec, sharex=sharex, sharey=sharey)
def _plot(self, ax, (behavior, scaling), error_type):
self.cs = plot_final_errors(ax, experiment, behavior, scaling, kappas, gammas, error_type)
p = ParamSearchPlot(behaviors, error_types, SubplotSpec(grid, 0), hspace=0.25, wspace=0.1)
p.axes.set_xticks([0.1, 0.5, 1.0, 1.5, 2.0])
for ax in p.axes_by_row[:-1]:
plt.setp(ax.get_xticklabels(), visible=False)
for ax in p.axes_by_col[1:]:
plt.setp(ax.get_yticklabels(), visible=False)
p.axes_by_row[-1].set_xlabel(r'$\kappa$', labelpad=0)
p.axes_by_col[0].set_ylabel(r'$\gamma$', rotation='horizontal', verticalalignment='center')
cb = fig.colorbar(p.cs, orientation='horizontal', cax=fig.add_subplot(grid[1]))
cb.ax.invert_xaxis()
cb.ax.set_xlabel('Normalized error', labelpad=0)
return fig
In [6]:
print_min_error('SingleSourceGaussian', 'DUCB', 1, kappas, gammas, 'rmse')
print_min_error('SingleSourceGaussian', 'DUCB', 'auto', kappas, gammas, 'rmse')
print_min_error('SingleSourceGaussian', 'PDUCB', 70, kappas, gammas, 'rmse')
print_min_error('SingleSourceGaussian', 'PDUCB', 'auto', kappas, gammas, 'rmse')
print_min_error('SingleSourceGaussian', 'GO', '', kappas, gammas, 'rmse')
print_min_error('SingleSourceGaussian', 'DUCB', 1, kappas, gammas, 'wrmse')
print_min_error('SingleSourceGaussian', 'DUCB', 'auto', kappas, gammas, 'wrmse')
print_min_error('SingleSourceGaussian', 'PDUCB', 70, kappas, gammas, 'wrmse')
print_min_error('SingleSourceGaussian', 'PDUCB', 'auto', kappas, gammas, 'wrmse')
print_min_error('SingleSourceGaussian', 'GO', '', kappas, gammas, 'wrmse')
In [9]:
print_min_error('SingleSourceGaussianDispersionNoiseless', 'DUCB', 1, kappas, gammas, 'rmse')
print_min_error('SingleSourceGaussianDispersionNoiseless', 'DUCB', 'auto', kappas, gammas, 'rmse')
print_min_error('SingleSourceGaussianDispersionNoiseless', 'PDUCB', 70, kappas, gammas, 'rmse')
print_min_error('SingleSourceGaussianDispersionNoiseless', 'PDUCB', 'auto', kappas, gammas, 'rmse')
print_min_error('SingleSourceGaussianDispersionNoiseless', 'GO', '', kappas, gammas, 'rmse')
print_min_error('SingleSourceGaussianDispersionNoiseless', 'DUCB', 1, kappas, gammas, 'wrmse')
print_min_error('SingleSourceGaussianDispersionNoiseless', 'DUCB', 'auto', kappas, gammas, 'wrmse')
print_min_error('SingleSourceGaussianDispersionNoiseless', 'PDUCB', 70, kappas, gammas, 'wrmse')
print_min_error('SingleSourceGaussianDispersionNoiseless', 'PDUCB', 'auto', kappas, gammas, 'wrmse')
print_min_error('SingleSourceGaussianDispersionNoiseless', 'GO', '', kappas, gammas, 'wrmse')
In [10]:
print_min_error('MultiSourceGaussianDispersionNoiseless', 'DUCB', 1, kappas, gammas, 'rmse')
print_min_error('MultiSourceGaussianDispersionNoiseless', 'DUCB', 'auto', kappas, gammas, 'rmse')
print_min_error('MultiSourceGaussianDispersionNoiseless', 'PDUCB', 70, kappas, gammas, 'rmse')
print_min_error('MultiSourceGaussianDispersionNoiseless', 'PDUCB', 'auto', kappas, gammas, 'rmse')
print_min_error('MultiSourceGaussianDispersionNoiseless', 'GO', '', kappas, gammas, 'rmse')
print_min_error('MultiSourceGaussianDispersionNoiseless', 'DUCB', 1, kappas, gammas, 'wrmse')
print_min_error('MultiSourceGaussianDispersionNoiseless', 'DUCB', 'auto', kappas, gammas, 'wrmse')
print_min_error('MultiSourceGaussianDispersionNoiseless', 'PDUCB', 70, kappas, gammas, 'wrmse')
print_min_error('MultiSourceGaussianDispersionNoiseless', 'PDUCB', 'auto', kappas, gammas, 'wrmse')
print_min_error('MultiSourceGaussianDispersionNoiseless', 'GO', '', kappas, gammas, 'wrmse')
In [46]:
fig = create_final_error_matrix(
experiment='SingleSourceGaussian',
behaviors=[('DUCB', 1), ('DUCB', 'auto'), ('PDUCB', 70), ('PDUCB', 'auto'), ('GO', '')],
error_types=['rmse', 'wrmse', 'rewards'])
In [47]:
fig.savefig('../../thesis/plots/psearch-G-NF-SS-SV.pdf')
In [48]:
fig = create_final_error_matrix(
experiment='SingleSourceGaussianDispersionNoiseless',
behaviors=[('DUCB', 1), ('DUCB', 'auto'), ('PDUCB', 70), ('PDUCB', 'auto'), ('GO', '')],
error_types=['rmse', 'wrmse', 'rewards'])
In [49]:
fig.savefig('../../thesis/plots/psearch-D-NF-SS-SV.pdf')
In [50]:
fig = create_final_error_matrix(
experiment='MultiSourceGaussianDispersionNoiseless',
behaviors=[('DUCB', 1), ('DUCB', 'auto'), ('PDUCB', 70), ('PDUCB', 'auto'), ('GO', '')],
error_types=['rmse', 'wrmse', 'rewards'])
In [51]:
fig.savefig('../../thesis/plots/psearch-D-NF-MS-SV.pdf')
In [6]:
@cached
def get_error_trace(experiment, paramstr, error_type):
datasets = []
for filename in iter_trial_filenames(os.path.join(experiment, paramstr)):
with tables.open_file(filename) as fileh:
d = fileh.get_node(fileh.root, error_type).read()
d /= d[0]
datasets.append(d)
if len(datasets) <= 0:
return np.ones(3000)
datasets = [extend_data(d, 3000) for d in datasets]
return np.mean(datasets, axis=0)
def plot_error_trace(ax, experiment, paramstr, error_type, **kwargs):
ax.plot(get_error_trace(experiment, paramstr, error_type), **kwargs)
experiment_labels = {
'SingleSourceGaussian': 'G-NF-SS-SV',
'SingleSourceGaussianDispersionNoiseless': 'D-NF-SS-SV',
'MultiSourceGaussianDispersionNoiseless': 'D-NF-MS-SV'
}
def create_error_trace_matrix(experiments, behaviors, error_type, figsize=(12, 12), legend_height=0.15):
fig = plt.figure(figsize=figsize)
grid = GridSpec(
2, 1, height_ratios=(1.0 - legend_height, legend_height), hspace=0.5,
left=0.1, right=0.975, bottom=0.15, top=0.87)
pal = SequentialLCPalette('seq', 210, (30, 80), (135, 30)).discretize(len(gammas))
hs = np.linspace(10, 360, len(pal))
for i in range(len(pal)):
pal[i] = MplLCHColor(pal[i].luminance, pal[i].chroma, hs[(i + 5) % len(pal)])
class ParamSearchPlot(SharedAxesGrid):
def _create_axes(self, subplot_spec, sharex, sharey):
return fig.add_subplot(subplot_spec, sharex=sharex, sharey=sharey)
def _plot(self, ax, experiment, (behavior, scaling)):
for j, gamma in enumerate(gammas):
cur_et = None
for i, kappa in enumerate(kappas):
if behavior == 'GO' and i != len(kappas) - 1:
continue
name = to_paramstr(behavior, scaling, kappa, gamma)
et = get_error_trace(experiment, name, error_type)
if cur_et is None or cur_et[-1] > et[-1]:
cur_et = et
ax.plot(cur_et, c=pal[j])
if scaling == 'auto':
behavior_label = 'auto-scaled ' + behavior
else:
behavior_label = behavior
ax.set_title(behavior_label)
p = ParamSearchPlot(experiments, behaviors, SubplotSpec(grid, 0))
latexstyle.style_axes(p.axes)
p.axes.xaxis.set_major_locator(plt.MaxNLocator(3))
for ax in p.axes_by_row[:-1]:
plt.setp(ax.get_xticklabels(), visible=False)
for ax in p.axes_by_col[1:]:
plt.setp(ax.get_yticklabels(), visible=False)
p.axes.set_ylim(0, 1.05)
p.axes_by_row[-1].set_xlabel('Simulation time [s]', labelpad=2)
p.axes_by_col[0].set_ylabel('Normalized RMISE', labelpad=3)
ax_legend = fig.add_subplot(grid[1])
plot_colors(pal, pos=(0.0, 0.0), axes=ax_legend)
#ax_legend.set_aspect(0.25)
ax_legend.set_xlim(0, len(gammas))
ax_legend.set_ylim(0.0, 1)
ax_legend.set_yticks([])
ax_legend.set_xticklabels([])
#ax_legend.set_xticks([])
ax2 = ax_legend.twiny()
ax2.xaxis.set_ticks_position('bottom')
ax2.set_xlim(0, len(gammas))
ax2.set_xticks(np.arange(len(gammas)) + 0.5)
ax2.tick_params(which='major', length=0)
ax2.set_xticklabels([r'$\mathdefault{0^{}}$'] + [r'$\mathdefault{{10^{{{}}}}}$'.format(x) for x in [-9, -8, -7, -6, -5, -4, -3, -2]])
ax2.xaxis.set_label_position('bottom')
ax2.set_xlabel(r'$\gamma$', labelpad=0)
#ax2.set_xticks(np.arange(len(gammas)) + 0.5)
#ax2.tick_params(length=0)
#ax2.set_xticklabels(['{:g}'.format(float(g)) for g in gammas])
#ax2.xaxis.set_major_formatter(LogFormatter())
#ax2.set_xlim(0.5, -5e-2)
#ax_legend.set_xticks(range(len(gammas)))
#ax_legend.set_xticks(np.arange(-0.5, len(gammas) - 0.5), minor=True)
#ax_legend.tick_params(which='major', length=0)
#ax_legend.tick_params(which='minor', length=rcParams['xtick.major.size'])
#sfmt = LogFormatter()
#sfmt.set_scientific(False)
#ax_legend.set_xticklabels(['0'] + [r'$-10^{{{}}}$'.format(x) for x in [-9, -8, -7, -6, -5, -4, -3, -2]])
#ax_legend.set_xticklabels(['0'] + [r'$\mathsf{{-10}}^{{{}}}}}$'.format(g) for g in [-9, -8, -7, -6, -5, -4, -3, -2]])
return fig
In [7]:
fig = create_error_trace_matrix(
experiments=['SingleSourceGaussian'],
behaviors=[('DUCB', 1), ('DUCB', 'auto'), ('PDUCB', 70), ('PDUCB', 'auto'), ('GO', '')],
error_type='rmse',
figsize=(5.35, 2.0), legend_height=0.1)
In [8]:
fig.savefig('../../thesis/plots/errtrace-nf.pdf')
In [14]:
create_error_trace_matrix(
experiment='SingleSourceGaussian',
behaviors=[('DUCB', 1), ('DUCB', 'auto'), ('PDUCB', 70), ('PDUCB', 'auto'), ('GO', '')],
error_types=['rmse', 'wrmse', 'rewards'])
In [15]:
create_error_trace_matrix(
experiment='SingleSourceGaussianDispersionNoiseless',
behaviors=[('DUCB', 1), ('DUCB', 'auto'), ('PDUCB', 70), ('PDUCB', 'auto'), ('GO', '')],
error_types=['rmse', 'wrmse', 'rewards'])
In [16]:
create_error_trace_matrix(
experiment='MultiSourceGaussianDispersionNoiseless',
behaviors=[('DUCB', 1), ('DUCB', 'auto'), ('PDUCB', 70), ('PDUCB', 'auto'), ('GO', '')],
error_types=['rmse', 'wrmse', 'rewards'])
In [8]: