In [1]:
%load_ext autoreload
%autoreload 2
In [2]:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import glob
import tabulate
import pprint
import click
import numpy as np
import pandas as pd
from ray.tune.commands import *
from nupic.research.frameworks.dynamic_sparse.common.browser import *
import matplotlib.pyplot as plt
from matplotlib import rcParams
%config InlineBackend.figure_format = 'retina'
import seaborn as sns
sns.set(style="whitegrid")
sns.set_palette("colorblind")
In [3]:
base = 'MLPHeb-non-binary-coacts-2019-10-04'
exps = [
os.path.join(base, exp) for exp in [
'mlp-heb',
'mlp-SET',
'mlp-WeightedMag',
'mlp-sparse',
]
]
paths = [os.path.expanduser("~/nta/results/{}".format(e)) for e in exps]
df = load_many(paths)
In [4]:
df.head(5)['hebbian_prune_perc']
Out[4]:
In [5]:
# replace hebbian prine
df['hebbian_prune_perc'] = df['hebbian_prune_perc'].replace(np.nan, 0.0, regex=True)
df['weight_prune_perc'] = df['weight_prune_perc'].replace(np.nan, 0.0, regex=True)
In [6]:
df.columns
Out[6]:
In [7]:
df.shape
Out[7]:
In [8]:
df.iloc[1]
Out[8]:
In [9]:
df.groupby('model')['model'].count()
Out[9]:
Experiment Details
In [10]:
# Did any trials failed?
df[df["epochs"]<30]["epochs"].count()
Out[10]:
In [11]:
# Removing failed or incomplete trials
df_origin = df.copy()
df = df_origin[df_origin["epochs"]>=30]
df.shape
Out[11]:
In [12]:
# which ones failed?
# failed, or still ongoing?
df_origin['failed'] = df_origin["epochs"]<30
df_origin[df_origin['failed']]['epochs']
Out[12]:
In [13]:
# helper functions
def mean_and_std(s):
return "{:.3f} ± {:.3f}".format(s.mean(), s.std())
def round_mean(s):
return "{:.0f}".format(round(s.mean()))
stats = ['min', 'max', 'mean', 'std']
def agg(columns, filter=None, round=3):
if filter is None:
return (df.groupby(columns)
.agg({'val_acc_max_epoch': round_mean,
'val_acc_max': stats,
'model': ['count']})).round(round)
else:
return (df[filter].groupby(columns)
.agg({'val_acc_max_epoch': round_mean,
'val_acc_max': stats,
'model': ['count']})).round(round)
In [14]:
agg(['model'])
Out[14]:
In [15]:
agg(['on_perc'])
Out[15]:
In [16]:
def model_name(row):
if row['model'] == 'DSNNWeightedMag':
return 'DSNN-WM'
elif row['model'] == 'DSNNMixedHeb':
if row['hebbian_prune_perc'] == 0.3:
return 'SET'
elif row['weight_prune_perc'] == 0.3:
return 'DSNN-Heb'
elif row['model'] == 'SparseModel':
return 'Static'
assert False, "This should cover all cases. Got {} h - {} w - {}".format(row['model'], row['hebbian_prune_perc'], row['weight_prune_perc'])
def test(val):
print(type(val))
print(val)
print()
In [17]:
df['model2'] = df.apply(model_name, axis=1)
In [18]:
fltr = df['model2'] != 'Sparse'
agg(['on_perc', 'model2'], filter=fltr)
Out[18]:
In [19]:
# translate model names
rcParams['figure.figsize'] = 16, 8
# d = {
# 'DSNNWeightedMag': 'DSNN',
# 'DSNNMixedHeb': 'SET',
# 'SparseModel': 'Static',
# }
# df_plot = df.copy()
# df_plot['model'] = df_plot['model'].apply(lambda x, i: model_name(x, i))
In [20]:
# sns.scatterplot(data=df_plot, x='on_perc', y='val_acc_max', hue='model')
sns.lineplot(data=df, x='on_perc', y='val_acc_max', hue='model2')
Out[20]:
In [22]:
rcParams['figure.figsize'] = 16, 8
filter = df['model'] != 'Static'
sns.lineplot(data=df[filter], x='on_perc', y='val_acc_max_epoch', hue='model2')
Out[22]:
In [24]:
sns.lineplot(data=df, x='on_perc', y='val_acc_last', hue='model2')
Out[24]:
In [ ]:
In [ ]: