In [ ]:
import sys, os
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import pandas as pd
import seaborn as sns
from ipywidgets import widgets
import ConfigReader as cr
import itertools as itt
import natsort as ns
import glob
In [ ]:
#Styles
sns.set_style('whitegrid', {'axes.linewidth':1.25, 'axes.edgecolor':'0.15',
'grid.linewidth':1.5, 'grid.color':'gray'})
sns.set_color_codes()
plt.rcParams['figure.figsize'] = (12.0, 9.0)
plt.rc('text', usetex=False)
plt.rc('font', size=20.0, family='serif')
markers = itt.cycle(('o', 's', 'p','H'))
In [ ]:
datasets_list =["1049", "1111", "1120", "1128", "179", "184", "293", "389", "38", "46", "554", "772", "917"]
In [ ]:
# Data location and scenario
data_dir = '/mhome/mendozah/autonet_GPU/results/experiment'
dataset = '1128_bac'
preprocessor='DeepNetIterative'
reader = cr.ConfigReader(data_dir=data_dir, dataset=dataset)
tdf = reader.load_validation_trajectories(preprocessor=preprocessor, load_config=False)
In [ ]:
# Data location and scenario
data_dir = '/mhome/mendozah/autonet_GPU/results/experiment'
dataset = '46_bac'
preprocessor='DeepNetIterative'
reader = cr.ConfigReader(data_dir=data_dir, dataset=dataset)
tdf = reader.load_validation_trajectories(preprocessor=preprocessor, load_config=False)
In [ ]:
temp_df = tdf.copy()
temp_df.columns = tdf.columns.droplevel(0)
tdf = temp_df.sort_values(by='time').drop_duplicates()
In [ ]:
data_dir = '/mhome/mendozah/autonet_full/results/experiment'
reader_cpu = cr.ConfigReader(data_dir=data_dir, dataset=dataset)
tdf_cpu = reader_cpu.load_validation_trajectories(preprocessor=preprocessor, load_config=False)
temp_cpu = tdf_cpu.copy()
temp_cpu.columns = tdf_cpu.columns.droplevel(0)
tdf_cpu = temp_cpu.sort_values(by='time').drop_duplicates()
In [ ]:
color_list = itt.cycle(['b', 'g','r', 'k'])
In [ ]:
def plot_trajectory_mean(df, ex, ax):
expand_mean_test = df.test_performance.cummean()
expand_mean_train = df.train_performance.cummean()
q1_test = expand_mean_test - expand_mean_test.std()
q3_test = expand_mean_test + expand_mean_test.std()
ax.plot(df.time, expand_mean_test, label=ex + '_test', marker=markers.next())
ax.plot(df.time, expand_mean_train, label=ex + '_train', marker=markers.next())
#ax.fill_between(df.time, q1_test, q3_test, alpha=0.4, color=color_list.next())
q1_train = expand_mean_train - expand_mean_train.std()
q3_train = expand_mean_train + expand_mean_train.std()
#ax.fill_between(df.time, q1_train, q3_train, alpha=0.4, color=color_list.next())
return ax
In [ ]:
fig_trajectory, ax_trajectory = plt.subplots(1,1)
for name, dataframe in zip(['gpu', 'cpu'], [tdf, tdf_cpu]):
ax_trajectory = plot_trajectory_mean(dataframe, name, ax_trajectory)
ax_trajectory.legend()
ax_trajectory.set_xscale('log')
In [ ]:
from smac_validate.plot_methods import plot_optimization_trace_mult_exp
from smac_validate.merge_test_performance_different_times import fill_trajectory
In [ ]:
preprocessor='DeepNetIterative'
In [ ]:
datasets_list =["1049", "1111", "1120", "1128", "179", "184", "293", "389", "38", "46", "554", "772", "917"]
In [ ]:
def read_validation_trajectories_csv(datadir, dataset, preprocessor):
ensemble_fn = 'validationResults-detailed-traj-run-*-walltime.csv'
scenario_dir = os.path.join(datadir, dataset + "_bac", preprocessor, dataset + "_bac", ensemble_fn)
traj_cols = ['time', 'train_performance', 'test_performance']
print(scenario_dir)
dirs = ns.natsorted(glob.glob(scenario_dir))
seeds = ['seed_' + itseeds.split('-')[-2].split('.')[0] for itseeds in dirs]
all_trajs = []
all_times = []
all_train_errors = []
all_test_errors = []
for fname in dirs:
try:
traj_res = pd.read_csv(fname, delimiter=",", usecols=[0,1,2],
skiprows=0)
except OSError:
print('file %s does not exist. Please check path' % fname)
except IndexError:
print('CRASH in: ' + os.path.split(fnames)[1])
except ValueError:
continue
traj_res = traj_res.apply(pd.to_numeric, errors='coerce')
traj_res.columns = traj_cols
traj_res.sort_values(by='time')
all_trajs.append(traj_res)
all_times.append(traj_res['time'].values)
all_train_errors.append(traj_res['train_performance'].values)
all_test_errors.append(traj_res['test_performance'].values)
trajectories_df = pd.concat(all_trajs, axis=1, keys=seeds)
return all_times, all_train_errors, all_test_errors
In [ ]:
properties = {"labelfontsize": 24,
"legendsize": 15,
"linewidth": 2.2,
"gridalpha": 0.5,
"gridcolor": "k",
"dpi": 300,
"colors": itt.cycle(["red", "blue"])}
In [ ]:
#experiments = ['full', 'GPU', 'ROAR']
#experiments = ['GPU']
#experiments = ['full', 'GPU']
experiments = ['full', '0305_benchmark']
plt.rcParams['figure.figsize'] = (12.0, 9.0)
In [ ]:
# Data location
dir_to_save = '/mhome/mendozah/autonet_full/results/plots_experiments/trajectory_plots/'
prepros=['DeepNetIterative', 'DeepFeedNet']
markers = itt.cycle(['o', 'v', '^', '8', 's', 'p', 'h', 'D'])
In [ ]:
for k, dataset in enumerate(datasets_list):
time_ = []
te_ = []
for j, (experiment, preprocessor) in enumerate(zip(experiments,prepros)):
datadir = "/mhome/mendozah/autonet_" + experiment + "/results/experiment"
ti, err_train, err_test = read_validation_trajectories_csv(datadir=datadir, dataset=dataset,
preprocessor=preprocessor)
#time_.append(ti)
#te_.append(err_train)
time_.append(ti)
te_.append(err_test)
te = []
time = []
for i in range(len(te_)):
p_, t_ = fill_trajectory(te_[i], time_[i])
time.append(t_[1:])
te.append([j[1:] for j in p_])
plot_optimization_trace_mult_exp(time, te, ["AutoNet-CPU-48-Test", "AutoNet-CPU-24-Test"],
logx=True, logy=False, title="", legend_loc=0,
scale_std=1, properties=properties, agglomeration='mean', ylabel="Error",
save = dir_to_save + "plot_compare_"+ dataset+"_time.pdf")
In [ ]: