In [1]:
%pylab inline
%load_ext autoreload
%autoreload 2
%run fix_paths.ipy
In [2]:
from glob import glob
import os
from srp.config import C
import pandas as pd
In [3]:
experiment_root =os.path.join(C.DATA, 'experiments')
roots = [exp for exp in os.listdir(experiment_root) if not exp.startswith(('.', '_'))]
In [58]:
fusion_root = 'data/experiments/fusion/'
obb_root = 'data/experiments/obb/'
synthetic_root = 'data/experiments/synthetic/'
cdrop_root = 'data/experiments/cdrop/'
regression_root = 'data/experiments/regression_loss/'
In [5]:
def check_progress():
files = glob(os.path.join(experiment_root, '*/*/*.csv'))
print ("{} files in total".format(len(files)))
notdone = []
for f in files:
df = pd.read_csv(f)
if len(df.columns) == 12:
doness = 'done'
else:
notdone.append(f)
doness = 'NOPE!'
#print ("{} \t {}".format(f, doness))
print("{} to go.".format(len(notdone)))
return notdone
In [6]:
notdone = check_progress()
In [7]:
def output_mean_variance_table(root):
index_label = []
rows = []
col_names = None
for subdir in os.listdir(root):
if subdir.startswith(('.', '_')):
continue
index_label.append(subdir)
csvs = glob(os.path.join(root, subdir, '*.csv'))
metrics = []
for csv in csvs:
if not col_names:
col_names = pd.read_csv(csv).columns[1:].tolist()
history = pd.read_csv(csv)
metrics.append(history.iloc[len(history)-5, 1:].values)
# print (index_label)
mean = np.mean(metrics, axis=0)
std = np.std(metrics, axis=0)
rows.append(["{:.3f}±{:.3f}".format(m,s) for (m, s) in zip(mean, std)])
return pd.DataFrame(data=np.array(rows), columns=col_names, index=index_label)
In [9]:
pd.read_csv('/home/liux13/srp/srp-boxes/results_c1r1/summary_fusion_c1r1.csv')
Out[9]:
In [99]:
output_mean_variance_table(os.path.join(experiment_root, roots[1]))
Out[99]:
In [64]:
output_mean_variance_table(os.path.join(experiment_root, roots[2]))
In [101]:
output_mean_variance_table(os.path.join(experiment_root, roots[3]))
Out[101]:
In [102]:
output_mean_variance_table(os.path.join(experiment_root, roots[4]))
Out[102]:
In [53]:
def output_folds_table(root):
csvs = glob(os.path.join(root, '*/*.csv'))
useful_cols = pd.read_csv(csvs[0]).columns[1:].tolist()
rows = []
index_label = []
for exp in csvs:
history = pd.read_csv(exp)
assert history.columns[1:].tolist() == useful_cols
rows.append(history.iloc[len(history)-5, 1:].values)
index_label.append(os.path.basename(os.path.dirname(exp))+'_'+os.path.basename(exp)[-5:-4])
assert len(useful_cols) == len(rows[0])
return pd.DataFrame(data=np.array(rows), columns=useful_cols, index=index_label)
In [57]:
output_folds_table(os.path.join(experiment_root, roots[0]))
Out[57]:
In [13]:
results_root = '/home/liux13/srp/srp-boxes/results_c1r1/'
raw_folds = []
mean_variance = []
for csv_path in glob(os.path.join(results_root, "*.csv")):
basename = os.path.basename(csv_path)
if basename.startswith('folds'):
raw_folds.append(csv_path)
else:
mean_variance.append(csv_path)
assert len(raw_folds) == len(mean_variance)
In [11]:
C.EXPERIMENT
Out[11]:
In [31]:
df = pd.read_csv(raw_folds[0])
for csv in raw_folds[1:]:
df = df.append(pd.read_csv(csv))
In [65]:
defaults = ('early', 'vector_and_width', 'no_pretrain', 'cdrop', 'smooth_L1')
In [75]:
df.columns = ['experiment', 'epoch', 'trn_loss', 'val_loss', 'f_2', 'accuracy',
'precision', 'recall', 'f_1', 'iou', 'L_1', 'L_2']
In [106]:
df.index = range(len(df))
In [111]:
def by(idx):
# print (idx)
param = df.iloc[idx]['experiment'][:-2]
if param in defaults:
return 'default'
else:
# print(param)
return param
In [117]:
means = df.groupby(by=by).mean()
stds = df.groupby(by=by).std()
In [130]:
stds['epoch']
Out[130]:
In [135]:
means.index.tolist()
Out[135]:
In [141]:
assert means.columns.tolist() == stds.columns.tolist()
assert means.index.tolist() == stds.index.tolist()
In [155]:
figsize(8, len(means.columns.tolist()) * 5)
for row, att in enumerate(means.columns.tolist()):
ax = plt.subplot(len(means.columns.tolist()),1,row+1)
ax.set_title(att)
plt.errorbar(means.index.tolist(), means[att], stds[att], linestyle='None', marker='^')
In [ ]: