In [1]:
%load_ext autoreload
%autoreload 2
In [2]:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import glob
import tabulate
import pprint
import click
import numpy as np
import pandas as pd
from ray.tune.commands import *
from nupic.research.frameworks.dynamic_sparse.common.browser import *
import matplotlib.pyplot as plt
from matplotlib import rcParams
from scipy.ndimage.filters import gaussian_filter1d
%config InlineBackend.figure_format = 'retina'
import seaborn as sns
sns.set(style="whitegrid")
sns.set_palette("colorblind")
In [8]:
exps = ['comparison_pruning_2' , 'comparison_iterative_pruning_2', 'comparison_set_2']
paths = [os.path.expanduser("~/nta/results/{}".format(e)) for e in exps]
df = load_many(paths)
In [9]:
df.head(5)
Out[9]:
In [10]:
df.shape
Out[10]:
In [11]:
df.columns
Out[11]:
In [17]:
df['model'].unique()
Out[17]:
In [24]:
# calculate density for each model
df.loc[df['model'] == 'PruningModel', 'density'] = df.loc[df['model'] == 'PruningModel', 'target_final_density']
df.loc[df['model'] == 'IterativePruningModel', 'density'] = df.loc[df['model'] == 'IterativePruningModel', 'target_final_density']
df.loc[df['model'] == 'SET', 'density'] = df.loc[df['model'] == 'SET', 'on_perc']
Experiment Details
In [25]:
# Did any trials failed?
num_epochs = 200
df[df["epochs"]<num_epochs]["epochs"].count()
Out[25]:
In [26]:
# Removing failed or incomplete trials
df_origin = df.copy()
df = df_origin[df_origin["epochs"]>=30]
df.shape
Out[26]:
In [27]:
# helper functions
def mean_and_std(s):
return "{:.3f} ± {:.3f}".format(s.mean(), s.std())
def round_mean(s):
return "{:.0f}".format(round(s.mean()))
stats = ['min', 'max', 'mean', 'std']
def agg(columns, filter=None, round=3):
if filter is None:
return (df.groupby(columns)
.agg({'val_acc_max_epoch': round_mean,
'val_acc_max': stats,
'model': ['count']})).round(round)
else:
return (df[filter].groupby(columns)
.agg({'val_acc_max_epoch': round_mean,
'val_acc_max': stats,
'model': ['count']})).round(round)
In [30]:
agg(['density', 'model'])
Out[30]:
In [36]:
# translate model names
rcParams['figure.figsize'] = 16, 8
sns.scatterplot(data=df, x='density', y='val_acc_max', hue='model')
sns.lineplot(data=df, x='density', y='val_acc_max', hue='model', legend=False);
In [ ]: