In [223]:
%load_ext autoreload
%autoreload 2
import os
import numpy as np
import pandas as pd
from browser import *
In [226]:
new_exps = [
'C100_DenseAdam',
'C100_DenseSGD',
'C100_SparseAdam',
'C100_SparseSGD',
'C10_DenseAdam',
'C10_DenseSGD',
'C10_SparseAdam',
'C10_SparseSGD',
]
old_exps = [
'VGG19DenseTest9v2',
'VGG19SparseFull',
'VGG19SparseFull-short',
'VGG19SparseTest9b2',
]
exps = new_exps + old_exps
In [227]:
paths = [os.path.expanduser("~/nta/results/{}".format(e)) for e in exps]
df = load_many(paths)
In [228]:
df.shape
Out[228]:
In [229]:
df.head()
Out[229]:
In [230]:
df.iloc[-1]
Out[230]:
In [ ]:
filters = (df['name'].str.startswith('C100_SparseSGD'))
(df[filters]
[['start_learning_rate', 'end_learning_rate', 'early_stop']]
.head(20))
In [243]:
filters = (df['name'].str.startswith('C100_SparseAdam'))
(df[filters]
[['start_learning_rate', 'end_learning_rate']]
.head(20))
Out[243]:
In [234]:
df['start_learning_rate'].max()
Out[234]:
In [235]:
df['end_learning_rate'].min()
Out[235]:
In [231]:
sum(df['early_stop']) / len(df)
Out[231]:
In [108]:
(df.groupby(['dataset', 'name'])['test_accuracy_max', 'test_accuracy', 'noise_accuracy_max',
'noise_accuracy', 'trial_time', 'batches_in_epoch',
'epochs']
.max().round(2))
Out[108]:
In [109]:
(df.groupby(['dataset', 'name'])['test_accuracy_max', 'test_accuracy', 'noise_accuracy_max',
'noise_accuracy', 'trial_time', 'batches_in_epoch',
'epochs']
.mean().round(2))
Out[109]:
In [64]:
(df.groupby(['dataset', 'name'])['test_accuracy_max', 'test_accuracy', 'noise_accuracy_max',
'noise_accuracy', 'experiment_time', 'batches_in_epoch']
.mean().round(3))
Out[64]:
In [73]:
metrics = ['epochs', 'test_accuracy_max', 'test_accuracy', 'noise_accuracy_max', 'noise_accuracy']
df[df['name'].str.startswith('C10_')][['name'] + metrics]
# (['dataset', 'name'])['test_accuracy_max', 'test_accuracy', 'noise_accuracy_max', 'noise_accuracy']
Out[73]:
In [138]:
metrics = ['epochs', 'test_accuracy', 'test_accuracy_max', 'noise_accuracy', 'noise_accuracy_max', 'early_stop']
df[df['name'].str.startswith('C100_DenseSGD')][['name'] + metrics].sort_values(['test_accuracy_max'], ascending=False)
Out[138]:
In [141]:
metrics = ['epochs', 'test_accuracy', 'test_accuracy_max', 'noise_accuracy', 'noise_accuracy_max', 'early_stop']
df[df['name'].str.startswith('C100_DenseAdam')][['name'] + metrics].sort_values(['test_accuracy_max'], ascending=False)
Out[141]:
In [220]:
df['noise_diff'] = df['noise_accuracy_max'] - df['noise_accuracy']
df['test_diff'] = df['test_accuracy_max'] - df['test_accuracy']
metrics = ['test_accuracy_max', 'test_accuracy', 'test_diff',
'noise_accuracy_max','noise_accuracy', 'noise_diff']
(df[df['name'].str.startswith('C100_SparseSGD')][metrics]
.sort_values(['test_accuracy_max'], ascending=False))
Out[220]:
In [219]:
df[metrics].mean()
Out[219]:
In [140]:
metrics = ['epochs', 'test_accuracy', 'test_accuracy_max', 'noise_accuracy', 'noise_accuracy_max', 'early_stop']
df[df['name'].str.startswith('C100_SparseSGD')][['name'] + metrics].sort_values(['test_accuracy_max'], ascending=False)
Out[140]:
In [142]:
metrics = ['epochs', 'test_accuracy', 'test_accuracy_max', 'noise_accuracy', 'noise_accuracy_max', 'early_stop']
df[df['name'].str.startswith('C100_DenseSGD')][['name'] + metrics].sort_values(['test_accuracy_max'], ascending=False)
Out[142]:
Can answer this question regarding:
In [148]:
tunable_params_general = ['learning_rate', 'learning_rate_gamma', 'weight_decay',
'momentum', 'batch_size', 'batches_in_epoch']
tunable_params_sparsity = ['boost_strength', 'boost_strength_factor',
'k_inference_factor', 'cnn_percent_on', 'cnn_weight_sparsity']
tunable_params = tunable_params_general + tunable_params_sparsity
performance_metrics = ['test_accuracy_max', 'mean_accuracy_max', 'noise_accuracy_max']
In [158]:
sparse_exps = [
'C100_SparseAdam',
'C100_SparseSGD',
'C10_SparseAdam',
'C10_SparseSGD',
'VGG19SparseFull',
'VGG19SparseFull-short',
'VGG19SparseTest9b2',
]
dense_exps = [
'VGG19DenseTest9v2',
'C100_DenseAdam',
'C100_DenseSGD',
'C10_DenseAdam',
'C10_DenseSGD',
]
In [149]:
performance_metrics = ['test_accuracy_max', 'mean_accuracy_max', 'noise_accuracy_max']
In [174]:
def stats(arr):
mean = np.mean(arr)
std = np.std(arr)
return [round(v, 4) for v in [mean-std, mean, mean+std]]
In [193]:
# CIFAR-10 SPARSE
filters = ((df['dataset']=='CIFAR10') &
(df['name'].isin(sparse_exps)) &
(df['optimizer'] != 'Adam'))
(df[filters]
.sort_values('mean_accuracy_max', ascending=False)[tunable_params]
.head(5)
.apply(stats))
Out[193]:
In [208]:
# CIFAR-10 SPARSE
filters = ((df['dataset']=='CIFAR10') &
(df['name'].isin(sparse_exps)) &
(df['optimizer'] != 'Adam'))
(df[filters]
.sort_values('mean_accuracy_max', ascending=False)[tunable_params]
.head(5)
.apply(stats))
Out[208]:
In [209]:
# CIFAR-100 SPARSE
filters = ((df['dataset']=='CIFAR100') &
(df['name'].isin(sparse_exps)) &
(df['optimizer'] != 'Adam'))
(df[filters]
.sort_values('mean_accuracy_max', ascending=False)[tunable_params]
.head(5)
.apply(stats))
Out[209]:
In [195]:
# CIFAR-100 SPARSE
filters = ((df['dataset']=='CIFAR100') &
(df['name'].isin(sparse_exps)) &
(df['optimizer'] != 'Adam'))
(df[filters]
.sort_values('mean_accuracy_max', ascending=False)[tunable_params]
.head(5)
.apply(stats))
Out[195]:
In [196]:
# CIFAR-100 DENSE
filters = ((df['dataset']=='CIFAR100') &
(df['name'].isin(dense_exps)) &
(df['optimizer'] != 'Adam'))
(df[filters]
.sort_values('mean_accuracy_max', ascending=False)[tunable_params]
.head(5)
.apply(stats))
Out[196]:
In [201]:
# CIFAR-100 SPARSE - VGG19SparseFull
filters = ((df['dataset']=='CIFAR100') &
(df['optimizer'] != 'Adam') &
(df['name'] == 'VGG19SparseFull'))
(df[filters]
.sort_values('mean_accuracy_max', ascending=False)[tunable_params]
.head(10)
.apply(stats))
Out[201]:
In [221]:
# CIFAR-100 SPARSE - VGG19SparseFull
filters = ((df['dataset']=='CIFAR100') &
(df['optimizer'] != 'Adam') &
(df['name'] == 'VGG19SparseFull'))
(df[filters]
.sort_values('test_accuracy_max', ascending=False)[tunable_params]
.head(10)
.apply(stats))
Out[221]:
In [222]:
# CIFAR-100 SPARSE - VGG19SparseFull
filters = ((df['dataset']=='CIFAR100') &
(df['optimizer'] != 'Adam') &
(df['name'] == 'VGG19SparseFull'))
(df[filters]
.sort_values('noise_accuracy_max', ascending=False)[tunable_params]
.head(10)
.apply(stats))
Out[222]:
In [210]:
metrics = ['epochs', 'mean_accuracy_max', 'test_accuracy_max', 'noise_accuracy_max']
(df[df['name'].str.startswith('VGG19SparseFull')][metrics]
.sort_values(['mean_accuracy_max'], ascending=False)
.iloc[:10])
Out[210]:
In [211]:
metrics = ['epochs', 'mean_accuracy_max', 'test_accuracy_max', 'noise_accuracy_max']
(df[df['name'].str.startswith('C100_DenseSGD')][metrics]
.sort_values(['mean_accuracy_max'], ascending=False)
.iloc[:10])
Out[211]:
In [250]:
filters = (df['name'].str.startswith('C100_SparseAdam'))
(df[filters]
[['start_learning_rate', 'end_learning_rate', 'early_stop', 'test_accuracy', 'epochs']]
.head(20))
Out[250]:
In [251]:
filters = (df['name'].str.startswith('C100_SparseSGD'))
(df[filters]
[['start_learning_rate', 'end_learning_rate', 'early_stop', 'test_accuracy', 'epochs']]
.head(20))
Out[251]:
In [ ]: