Experiment:

Replicate how so dense experiments using the new dynamic sparse framework. Compare results with published in the paper

Motivation.

  • Ensure our code has no known bugs before proceeding with further experimentation.
  • Ensure How so Dense experiments are replicable

Conclusion

base_exp_config = dict( device="cuda", # ----- dataset related ---- dataset_name="PreprocessedGSC", data_dir=os.path.expanduser("~/nta/datasets/gsc"), train_batches_per_epoch=5121, # batch_size_train=(4, 16), batch_size_train=16, batch_size_test=20, # required to fit the GPU # ----- network related ---- network="GSCHeb", percent_on_k_winner=[0.095, 0.125, 0.067], k_inference_factor=1.5, boost_strength=[1.5, 1.5, 1.5], boost_strength_factor=[0.9, 0.9, 0.9], hidden_neurons_conv=[64, 64], hidden_neurons_fc=1500, bias=True, dropout=False, batch_norm=True, # ----- model related ---- model=tune.grid_search( ["BaseModel", "SparseModel", "DSNNWeightedMag", "DSNNMixedHeb"] ), optim_alg="SGD", momentum=0, learning_rate=0.01, weight_decay=0.01, lr_scheduler="StepLR", lr_gamma=0.9, on_perc=[1, 1, 0.1, 1], hebbian_prune_perc=None, hebbian_grow=False, weight_prune_perc=0.3, pruning_early_stop=None, # 2 # additional validation test_noise=True, # debugging debug_weights=True, debug_sparse=True, ) # ray configurations tune_config = dict( name=__file__.replace(".py", "") + "_test2", num_samples=8, local_dir=os.path.expanduser("~/nta/results"), checkpoint_freq=0, checkpoint_at_end=False, stop={"training_iteration": 25}, resources_per_trial={"cpu": 1, "gpu": 0.25}, loggers=DEFAULT_LOGGERS, verbose=0, )

In [1]:
%load_ext autoreload
%autoreload 2

In [2]:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import glob
import tabulate
import pprint
import click
import numpy as np
import pandas as pd
from ray.tune.commands import *
from nupic.research.frameworks.dynamic_sparse.common.browser import *

import matplotlib
import matplotlib.pyplot as plt
from matplotlib import rcParams

%config InlineBackend.figure_format = 'retina'

import seaborn as sns
sns.set(style="whitegrid")
sns.set_palette("colorblind")

Load and check data


In [3]:
# exps = ['replicate_hsd_test2']
# exps = ['replicate_hsd_debug2']
exps = ['replicate_hsd_debug5_2x']
# exps = ['replicate_hsd_debug5_2x', 'replicate_hsd_debug6_8x']
paths = [os.path.expanduser("~/nta/results/{}".format(e)) for e in exps]
df = load_many(paths)

In [4]:
df.head(5)


Out[4]:
Experiment Name train_acc_max train_acc_max_epoch train_acc_min train_acc_min_epoch train_acc_median train_acc_last val_acc_max val_acc_max_epoch val_acc_min ... momentum network on_perc optim_alg percent_on_k_winner pruning_early_stop test_noise train_batches_per_epoch weight_decay weight_prune_perc
0 0_model=BaseModel 0.950298 23 0.636412 0 0.937262 0.950249 0.967121 24 0.893745 ... 0 GSCHeb 0.775 SGD 0.095667 None True 5121 0.01 0.3
1 1_model=SparseModel 0.948198 24 0.655502 0 0.933161 0.948198 0.962711 24 0.878508 ... 0 GSCHeb 0.775 SGD 0.095667 None True 5121 0.01 0.3
2 2_model=DSNNWeightedMag 0.948247 24 0.622107 0 0.932526 0.948247 0.965116 24 0.876103 ... 0 GSCHeb 0.775 SGD 0.095667 None True 5121 0.01 0.3
3 3_model=DSNNMixedHeb 0.950591 24 0.668245 0 0.934821 0.950591 0.961909 22 0.900561 ... 0 GSCHeb 0.775 SGD 0.095667 None True 5121 0.01 0.3
4 4_model=BaseModel 0.951470 24 0.634801 0 0.938629 0.951470 0.963111 23 0.887731 ... 0 GSCHeb 0.775 SGD 0.095667 None True 5121 0.01 0.3

5 rows × 55 columns


In [5]:
# replace hebbian prine
df['hebbian_prune_perc'] = df['hebbian_prune_perc'].replace(np.nan, 0.0, regex=True)
df['weight_prune_perc'] = df['weight_prune_perc'].replace(np.nan, 0.0, regex=True)

In [6]:
df.columns


Out[6]:
Index(['Experiment Name', 'train_acc_max', 'train_acc_max_epoch',
       'train_acc_min', 'train_acc_min_epoch', 'train_acc_median',
       'train_acc_last', 'val_acc_max', 'val_acc_max_epoch', 'val_acc_min',
       'val_acc_min_epoch', 'val_acc_median', 'val_acc_last', 'noise_acc_max',
       'noise_acc_max_epoch', 'noise_acc_min', 'noise_acc_min_epoch',
       'noise_acc_median', 'noise_acc_last', 'val_acc_all', 'epochs',
       'experiment_file_name', 'trial_time', 'mean_epoch_time', 'batch_norm',
       'batch_size_test', 'batch_size_train', 'bias', 'boost_strength',
       'boost_strength_factor', 'data_dir', 'dataset_name', 'debug_sparse',
       'debug_weights', 'device', 'dropout', 'hebbian_grow',
       'hebbian_prune_perc', 'hidden_neurons_conv', 'hidden_neurons_fc',
       'k_inference_factor', 'learning_rate', 'lr_gamma', 'lr_scheduler',
       'model', 'momentum', 'network', 'on_perc', 'optim_alg',
       'percent_on_k_winner', 'pruning_early_stop', 'test_noise',
       'train_batches_per_epoch', 'weight_decay', 'weight_prune_perc'],
      dtype='object')

In [7]:
df.shape


Out[7]:
(120, 55)

In [8]:
df.iloc[1]


Out[8]:
Experiment Name                                          1_model=SparseModel
train_acc_max                                                       0.948198
train_acc_max_epoch                                                       24
train_acc_min                                                       0.655502
train_acc_min_epoch                                                        0
train_acc_median                                                    0.933161
train_acc_last                                                      0.948198
val_acc_max                                                         0.962711
val_acc_max_epoch                                                         24
val_acc_min                                                         0.878508
val_acc_min_epoch                                                          0
val_acc_median                                                      0.952285
val_acc_last                                                        0.962711
noise_acc_max                                                       0.970219
noise_acc_max_epoch                                                       21
noise_acc_min                                                     0.00548589
noise_acc_min_epoch                                                        8
noise_acc_median                                                    0.172414
noise_acc_last                                                      0.119122
val_acc_all                0     0.878508
1     0.906576
2     0.916199
3...
epochs                                                                    25
experiment_file_name       /Users/lsouza/nta/results/replicate_hsd_debug5...
trial_time                                                           5.59045
mean_epoch_time                                                     0.223618
batch_norm                                                              True
batch_size_test                                                           20
batch_size_train                                                          16
bias                                                                    True
boost_strength                                                           1.5
boost_strength_factor                                                    0.9
data_dir                                       /home/ubuntu/nta/datasets/gsc
dataset_name                                                 PreprocessedGSC
debug_sparse                                                            True
debug_weights                                                           True
device                                                                  cuda
dropout                                                                False
hebbian_grow                                                           False
hebbian_prune_perc                                                         0
hidden_neurons_conv                                                       64
hidden_neurons_fc                                                       1500
k_inference_factor                                                       1.5
learning_rate                                                           0.01
lr_gamma                                                                 0.9
lr_scheduler                                                          StepLR
model                                                            SparseModel
momentum                                                                   0
network                                                               GSCHeb
on_perc                                                                0.775
optim_alg                                                                SGD
percent_on_k_winner                                                0.0956667
pruning_early_stop                                                      None
test_noise                                                              True
train_batches_per_epoch                                                 5121
weight_decay                                                            0.01
weight_prune_perc                                                        0.3
Name: 1, dtype: object

In [9]:
df.groupby('model')['model'].count()


Out[9]:
model
BaseModel          30
DSNNMixedHeb       30
DSNNWeightedMag    30
SparseModel        30
Name: model, dtype: int64

Analysis

Experiment Details


In [10]:
num_epochs = 25
# Did any  trials failed?
df[df["epochs"]<num_epochs]["epochs"].count()


Out[10]:
0

In [11]:
# Removing failed or incomplete trials
df_origin = df.copy()
df = df_origin[df_origin["epochs"]>=num_epochs]
df.shape


Out[11]:
(120, 55)

In [12]:
# which ones failed?
# failed, or still ongoing?
df_origin['failed'] = df_origin["epochs"]<num_epochs
df_origin[df_origin['failed']]['epochs']


Out[12]:
Series([], Name: epochs, dtype: int64)

In [13]:
# helper functions
def mean_and_std(s):
    return "{:.2f} ± {:.2f}".format(s.mean()*100, s.std()*100)

def round_mean(s):
    return "{:.0f}".format(round(s.mean()))

stats = ['min', 'max', 'mean', 'std']

def agg(columns, filter=None, round=3):
    if filter is None:
        return (df.groupby(columns)
             .agg({'val_acc_max_epoch': round_mean,
                   'val_acc_max': stats,
                   'val_acc_last': stats,
                   'model': ['count']})).round(round)
    else:
        return (df[filter].groupby(columns)
             .agg({'val_acc_max_epoch': round_mean,
                   'val_acc_max': stats,                
                   'val_acc_last': stats,
                   'model': ['count']})).round(round)
    
    
def agg_paper(columns, filter=None, round=3):
    if filter is None:
        return (df.groupby(columns)
             .agg({'val_acc_max': mean_and_std,
                   'val_acc_last': mean_and_std,
                   'train_acc_last': mean_and_std,
                   'model': ['count']})).round(round)
    else:
        return (df[filter].groupby(columns)
             .agg({'val_acc_max': mean_and_std,
                   'val_acc_last': mean_and_std,
                   'train_acc_last': mean_and_std,
                   'model': ['count']})).round(round)
Results

In [14]:
agg(['model'])


Out[14]:
val_acc_max_epoch val_acc_max val_acc_last model
round_mean min max mean std min max mean std count
model
BaseModel 22 0.962 0.970 0.965 0.002 0.958 0.967 0.962 0.002 30
DSNNMixedHeb 20 0.960 0.966 0.963 0.002 0.953 0.966 0.960 0.003 30
DSNNWeightedMag 20 0.961 0.968 0.964 0.002 0.954 0.965 0.961 0.003 30
SparseModel 21 0.958 0.966 0.963 0.002 0.956 0.966 0.961 0.003 30

In [15]:
agg_paper(['model'])


Out[15]:
val_acc_max val_acc_last train_acc_last model
mean_and_std mean_and_std mean_and_std count
model
BaseModel 96.52 ± 0.18 96.21 ± 0.23 95.14 ± 0.12 30
DSNNMixedHeb 96.32 ± 0.16 96.04 ± 0.28 94.91 ± 0.14 30
DSNNWeightedMag 96.37 ± 0.19 96.10 ± 0.31 94.89 ± 0.14 30
SparseModel 96.27 ± 0.17 96.07 ± 0.26 94.88 ± 0.16 30

In [ ]: