In [1]:
!pip install oyaml
In [2]:
""" Generate modules for all combinations of settings we plan to test.
"""
from collections import OrderedDict
import itertools
import pandas as pd
# List options in the order that you want to test.
# Put the hypothetical best values first; that way
# the first experments will assume the best values
# for all options
OPTIONS = OrderedDict() # for python2 compatability
OPTIONS['FUSION'] = ['early_fusion', 'late_fusion_add', 'late_fusion_cat']
OPTIONS['OBB_PARAMETRIZATION'] = [ 'vector_and_width', 'angle_length_width', 'four_points', 'two_vectors']
OPTIONS['SYNTHETIC'] = ['no_pretrain', 'pretrain']
OPTIONS['CHANNEL_DROPOUT'] = ['cdrop', 'no_cdrop']
OPTIONS['CLASS_LOSS'] = ['hinge_loss', 'xent_loss']
OPTIONS['REGRESSION_LOSS'] = ['smooth_L1', 'L2']
OPTIONS['AUG_JITTER'] = ['no_jitter', 'jitter']
COMBOS = [ reversed(combo) for combo in itertools.product(*reversed(OPTIONS.values()))]
In [6]:
ops = list(OPTIONS.values())
In [7]:
sum([len(ops[i])*len(ops[i+1]) for i in range(len(OPTIONS.values())-1)])
Out[7]:
In [ ]:
In [96]:
EXPERIMENTS = pd.DataFrame(COMBOS, columns=OPTIONS.keys())
EXPERIMENTS
Out[96]:
In [119]:
combo = EXPERIMENTS.iloc[10]
In [120]:
dict(combo)
Out[120]:
In [121]:
def save_options(f):
for k, v in combo.items():
print('{}: {}'.format(k,v), file=f)
import sys
save_options(sys.stdout)
In [171]:
import oyaml as yaml
rec = OrderedDict()
rec['EXPERIMENT'] = combo.name
rec.update(combo)
print(yaml.dump(rec, default_flow_style=False))
In [140]:
import os
def make_relpath(combo):
return '-'.join(('{:05}'.format(combo.name),)+ tuple(combo))
In [142]:
make_relpath(combo)
Out[142]:
In [178]:
INT_DATA = '../data/interim/'
In [179]:
print(yaml.dump(OPTIONS))
In [192]:
from tqdm import tqdm
In [209]:
import oyaml as yaml
def make_experiments(options):
# Save the options
with open(os.path.join(INT_DATA, 'options.yml'), 'w') as f:
yaml.dump(options, f, default_flow_style=False)
# Save the master list of experiments
combos = [ reversed(combo) for combo in itertools.product(*reversed(options.values()))]
experiments = pd.DataFrame(combos, columns=options.keys())
experiments.to_csv(os.path.join(INT_DATA, 'experiments.csv'))
# Make s folder and config file for each experiment
for i, combo in tqdm(experiments.iterrows(), "generating configs"):
dirname = '{:05}'.format(combo.name)
rec = OrderedDict()
rec['NUMBER'] = combo.name
rec['NAME'] = '-'.join((dirname,)+ tuple(combo))
rec.update(combo)
os.makedirs(os.path.join(INT_DATA, 'experiments', dirname), exist_ok=True)
with open(os.path.join(INT_DATA, 'experiments', dirname, 'config.yml'), 'w') as f:
yaml.dump(rec, f, default_flow_style=False)
In [210]:
make_experiments(OPTIONS)
In [ ]: