Experiment: test_restoration

Evaluate if restoration affected existing capabilities. Comparing two approaches to calculate coactivations to see if they are getting to the same values.

Conclusion

base_exp_config = dict( device="cuda", # dataset related dataset_name="PreprocessedGSC", data_dir="~/nta/datasets/gsc", batch_size_train=(4, 16), batch_size_test=1000, # network related network=tune.grid_search(["GSCHeb_v0", "GSCHeb"]), optim_alg="SGD", momentum=0, # 0.9, learning_rate=0.01, # 0.1, weight_decay=0.01, # 1e-4, lr_scheduler="MultiStepLR", lr_milestones=[30, 60, 90], lr_gamma=0.9, # 0.1, use_kwinners=True, # sparse_linear_only=True, # False # model related model="DSNNWeightedMag", # on_perc=0.04, # sparse related on_perc=tune.grid_search([0.02, 0.03, 0.04]), weight_prune_perc=0.3, # weight_prune_perc=tune.grid_search(list(np.arange(0, 1.001, 0.05))), # pruning_early_stop=2, # additional validation # test_noise=False, # debugging # debug_weights=True, # debug_sparse=False, )

In [1]:
%load_ext autoreload
%autoreload 2

In [2]:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import glob
import tabulate
import pprint
import click
import numpy as np
import pandas as pd
from ray.tune.commands import *
from nupic.research.frameworks.dynamic_sparse.common.browser import *

import matplotlib
import matplotlib.pyplot as plt
from matplotlib import rcParams

%config InlineBackend.figure_format = 'retina'

import seaborn as sns
sns.set(style="whitegrid")
sns.set_palette("colorblind")

Load and check data


In [9]:
exps = ['test_restoration_5']
paths = [os.path.expanduser("~/nta/results/{}".format(e)) for e in exps]
df = load_many(paths)

In [10]:
df.head(5)


Out[10]:
Experiment Name train_acc_max train_acc_max_epoch train_acc_min train_acc_min_epoch train_acc_median train_acc_last val_acc_max val_acc_max_epoch val_acc_min ... lr_milestones lr_scheduler model momentum network on_perc optim_alg use_kwinners weight_decay weight_prune_perc
0 0_network=GSCHeb_v0,on_perc=0.02 0.780295 62 0.248023 0 0.749780 0.733913 0.865277 63 0.138332 ... 60.0 MultiStepLR DSNNWeightedMag 0 GSCHeb_v0 0.02 SGD True 0.01 None-None-0.3-0.3
1 1_network=GSCHeb,on_perc=0.02 0.811298 46 0.241724 0 0.779709 0.766868 0.884523 19 0.309944 ... 60.0 MultiStepLR DSNNWeightedMag 0 GSCHeb 0.02 SGD True 0.01 None-None-0.3-0.3
2 2_network=GSCHeb_v0,on_perc=0.03 0.854799 60 0.263548 0 0.842301 0.839566 0.921010 60 0.384122 ... 60.0 MultiStepLR DSNNWeightedMag 0 GSCHeb_v0 0.03 SGD True 0.01 None-None-0.3-0.3
3 3_network=GSCHeb,on_perc=0.03 0.855239 70 0.263939 0 0.837394 0.846988 0.923817 96 0.141540 ... 60.0 MultiStepLR DSNNWeightedMag 0 GSCHeb 0.03 SGD True 0.01 None-None-0.3-0.3
4 4_network=GSCHeb_v0,on_perc=0.04 0.864515 91 0.281564 0 0.851723 0.859438 0.925421 65 0.336808 ... 60.0 MultiStepLR DSNNWeightedMag 0 GSCHeb_v0 0.04 SGD True 0.01 None-None-0.3-0.3

5 rows × 35 columns


In [ ]:
# replace hebbian prine
# df['hebbian_prune_perc'] = df['hebbian_prune_perc'].replace(np.nan, 0.0, regex=True)
# df['weight_prune_perc'] = df['weight_prune_perc'].replace(np.nan, 0.0, regex=True)

In [11]:
df.columns


Out[11]:
Index(['Experiment Name', 'train_acc_max', 'train_acc_max_epoch',
       'train_acc_min', 'train_acc_min_epoch', 'train_acc_median',
       'train_acc_last', 'val_acc_max', 'val_acc_max_epoch', 'val_acc_min',
       'val_acc_min_epoch', 'val_acc_median', 'val_acc_last', 'val_acc_all',
       'epochs', 'experiment_file_name', 'trial_time', 'mean_epoch_time',
       'batch_size_test', 'batch_size_train', 'data_dir', 'dataset_name',
       'device', 'learning_rate', 'lr_gamma', 'lr_milestones', 'lr_scheduler',
       'model', 'momentum', 'network', 'on_perc', 'optim_alg', 'use_kwinners',
       'weight_decay', 'weight_prune_perc'],
      dtype='object')

In [12]:
df.shape


Out[12]:
(60, 35)

In [13]:
df.iloc[1]


Out[13]:
Experiment Name                             1_network=GSCHeb,on_perc=0.02
train_acc_max                                                    0.811298
train_acc_max_epoch                                                    46
train_acc_min                                                    0.241724
train_acc_min_epoch                                                     0
train_acc_median                                                 0.779709
train_acc_last                                                   0.766868
val_acc_max                                                      0.884523
val_acc_max_epoch                                                      19
val_acc_min                                                      0.309944
val_acc_min_epoch                                                      81
val_acc_median                                                   0.846231
val_acc_last                                                     0.836006
val_acc_all             0     0.377706
1     0.630714
2     0.695269
3...
epochs                                                                100
experiment_file_name    /Users/lsouza/nta/results/test_restoration_5/e...
trial_time                                                        33.2763
mean_epoch_time                                                  0.332763
batch_size_test                                                      1000
batch_size_train                                                       10
data_dir                                               ~/nta/datasets/gsc
dataset_name                                              PreprocessedGSC
device                                                               cuda
learning_rate                                                        0.01
lr_gamma                                                              0.9
lr_milestones                                                          60
lr_scheduler                                                  MultiStepLR
model                                                     DSNNWeightedMag
momentum                                                                0
network                                                            GSCHeb
on_perc                                                              0.02
optim_alg                                                             SGD
use_kwinners                                                         True
weight_decay                                                         0.01
weight_prune_perc                                       None-None-0.3-0.3
Name: 1, dtype: object

In [14]:
df.groupby('model')['model'].count()


Out[14]:
model
DSNNWeightedMag    60
Name: model, dtype: int64

Analysis

Experiment Details


In [15]:
num_epochs=100

In [16]:
# Did any  trials failed?
df[df["epochs"]<num_epochs]["epochs"].count()


Out[16]:
0

In [17]:
# Removing failed or incomplete trials
df_origin = df.copy()
df = df_origin[df_origin["epochs"]>=num_epochs]
df.shape


Out[17]:
(60, 35)

In [18]:
# which ones failed?
# failed, or still ongoing?
df_origin['failed'] = df_origin["epochs"]<num_epochs
df_origin[df_origin['failed']]['epochs']


Out[18]:
Series([], Name: epochs, dtype: int64)

In [19]:
# helper functions
def mean_and_std(s):
    return "{:.3f} ± {:.3f}".format(s.mean(), s.std())

def round_mean(s):
    return "{:.0f}".format(round(s.mean()))

stats = ['min', 'max', 'mean', 'std']

def agg(columns, filter=None, round=3):
    if filter is None:
        return (df.groupby(columns)
             .agg({'val_acc_max_epoch': round_mean,
                   'val_acc_max': stats,                
                   'model': ['count']})).round(round)
    else:
        return (df[filter].groupby(columns)
             .agg({'val_acc_max_epoch': round_mean,
                   'val_acc_max': stats,                
                   'model': ['count']})).round(round)
Does improved weight pruning outperforms regular SET

In [20]:
agg(['on_perc', 'network'])


Out[20]:
val_acc_max_epoch val_acc_max model
round_mean min max mean std count
on_perc network
0.02 GSCHeb 37 0.864 0.889 0.875 0.008 10
GSCHeb_v0 42 0.865 0.887 0.876 0.006 10
0.03 GSCHeb 66 0.893 0.924 0.914 0.010 10
GSCHeb_v0 62 0.907 0.922 0.916 0.005 10
0.04 GSCHeb 81 0.923 0.937 0.933 0.004 10
GSCHeb_v0 73 0.923 0.938 0.930 0.004 10

In [ ]: