Experiment:

Evaluate pruning by magnitude weighted by coactivations (more thorough evaluation), compare it to baseline (SET).

Motivation.

Check if results are consistently above baseline.

Conclusion

No significant difference between both models
No support for early stopping



In [326]:

    
from IPython.display import Markdown, display
%load_ext autoreload
%autoreload 2









    



The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload



In [327]:

    
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import glob
import tabulate
import pprint
import click
import numpy as np
import pandas as pd
from ray.tune.commands import *
from nupic.research.frameworks.dynamic_sparse.common.browser import *

import matplotlib.pyplot as plt
from matplotlib import rcParams

%config InlineBackend.figure_format = 'retina'

import seaborn as sns
sns.set(style="whitegrid")
sns.set_palette("colorblind")

Load and check data



In [328]:

    
base = os.path.join('gsc-dsnn-2019-10-11-G-reproduce')
exps = [
    os.path.join(base, exp) for exp in [
#         'gsc-Static', 
        'gsc-SET',
        'gsc-WeightedMag',
    ]
]
paths = [os.path.expanduser("~/nta/results/{}".format(e)) for e in exps]
df = load_many(paths)









    



gsc-SET
gsc-WeightedMag



In [329]:

    
# replace hebbian prine
df['hebbian_prune_perc'] = df['hebbian_prune_perc'].replace(np.nan, 0.0, regex=True)
df['weight_prune_perc'] = df['weight_prune_perc'].replace(np.nan, 0.0, regex=True)
display(df.head(5)['hebbian_prune_perc'])
display(df.head(5)['weight_prune_perc'])









    





0    0.0
1    0.0
2    0.0
3    0.0
4    0.0
Name: hebbian_prune_perc, dtype: float64






    





0    None-None-0.3-0.3
1    None-None-0.3-0.3
2    None-None-0.3-0.3
3    None-None-0.3-0.3
4    None-None-0.3-0.3
Name: weight_prune_perc, dtype: object



In [330]:

    
df.iloc[200:205]









    Out[330]:







  
    
      
      Experiment Name
      train_acc_max
      train_acc_max_epoch
      train_acc_min
      train_acc_min_epoch
      train_acc_median
      train_acc_last
      val_acc_max
      val_acc_max_epoch
      val_acc_min
      ...
      model
      momentum
      net_params
      network
      on_perc
      optim_alg
      prune_methods
      test_noise
      weight_decay
      weight_prune_perc
    
  
  
  

0 rows × 43 columns



In [331]:

    
df.columns









    Out[331]:





Index(['Experiment Name', 'train_acc_max', 'train_acc_max_epoch',
       'train_acc_min', 'train_acc_min_epoch', 'train_acc_median',
       'train_acc_last', 'val_acc_max', 'val_acc_max_epoch', 'val_acc_min',
       'val_acc_min_epoch', 'val_acc_median', 'val_acc_last', 'val_acc_all',
       'epochs', 'experiment_file_name', 'experiment_base_path', 'trial_time',
       'mean_epoch_time', 'scatter_plot_dicts', 'batch_size_test',
       'batch_size_train', 'data_dir', 'dataset_name', 'debug_sparse',
       'debug_weights', 'device', 'hebbian_grow', 'hebbian_prune_perc',
       'learning_rate', 'lr_gamma', 'lr_milestones', 'lr_scheduler', 'model',
       'momentum', 'net_params', 'network', 'on_perc', 'optim_alg',
       'prune_methods', 'test_noise', 'weight_decay', 'weight_prune_perc'],
      dtype='object')



In [332]:

    
df.shape









    Out[332]:





(20, 43)



In [333]:

    
df.iloc[1]









    Out[333]:





Experiment Name              1_model=SET,on_perc=[0.04, 0.04, 0.04, 0.04]
train_acc_max                                                    0.852602
train_acc_max_epoch                                                    98
train_acc_min                                                    0.291378
train_acc_min_epoch                                                     0
train_acc_median                                                 0.825774
train_acc_last                                                   0.850991
val_acc_max                                                      0.914577
val_acc_max_epoch                                                      94
val_acc_min                                                      0.459639
val_acc_min_epoch                                                       0
val_acc_median                                                   0.882053
val_acc_last                                                     0.892633
val_acc_all             0     0.459639
1     0.687696
2     0.800157
3...
epochs                                                                100
experiment_file_name    /Users/mcaporale/nta/results/gsc-dsnn-2019-10-...
experiment_base_path                                              gsc-SET
trial_time                                                        15.5704
mean_epoch_time                                                  0.155704
scatter_plot_dicts                                                     {}
batch_size_test                                                      1000
batch_size_train                                                       10
data_dir                                               ~/nta/datasets/gsc
dataset_name                                              PreprocessedGSC
debug_sparse                                                         True
debug_weights                                                        True
device                                                               cuda
hebbian_grow                                                        False
hebbian_prune_perc                                                      0
learning_rate                                                        0.01
lr_gamma                                                              0.9
lr_milestones                                                          60
lr_scheduler                                                  MultiStepLR
model                                                                 SET
momentum                                                                0
net_params              {'boost_strength': 1.5, 'boost_strength_factor...
network                                                   gsc_sparse_dsnn
on_perc                                                              0.04
optim_alg                                                             SGD
prune_methods                                         None-None-None-None
test_noise                                                          False
weight_decay                                                         0.01
weight_prune_perc                                       None-None-0.3-0.3
Name: 1, dtype: object



In [334]:

    
df.groupby('model')['model'].count()









    Out[334]:





model
DSNNWeightedMag    10
SET                10
Name: model, dtype: int64

Analysis

Experiment Details

# experiment configurations base_exp_config = dict( device=("cuda" if torch.cuda.device_count() > 0 else "cpu"), # dataset related dataset_name="PreprocessedGSC", data_dir="~/nta/datasets/gsc", batch_size_train=16, batch_size_test=(1000), # network related network="GSCHeb", # ----- Optimizer Related ---- optim_alg="SGD", momentum=0.0, learning_rate=0.01, weight_decay=1e-2, # ----- LR Scheduler Related ---- lr_scheduler="StepLR", lr_step_size=1, lr_gamma=0.9, # additional validation test_noise=False, # debugging debug_weights=True, debug_sparse=True, ) # ray configurations # experiment_name = "gsc-trials-2019-10-07" experiment_name = "gsc-dsnn-2019-10-11" tune_config = dict( name=experiment_name, num_samples=5, local_dir=os.path.expanduser(os.path.join("~/nta/results", experiment_name)), checkpoint_freq=0, checkpoint_at_end=False, stop={"training_iteration": 30}, resources_per_trial={ "cpu": os.cpu_count() / torch.cuda.device_count(), "gpu": 1, }, loggers=DEFAULT_LOGGERS, verbose=1, config=base_exp_config, ) # define experiments on_perc_levels = np.arange(0, 0.101, 0.005) experiments = { "gsc-Static": dict( model=ray.tune.grid_search(["SparseModel"]), network="gsc_sparse_dsnn", # sparse related on_perc=ray.tune.grid_search([ [perc] * 4 for perc in on_perc_levels ]), ), "gsc-SET": dict( model=ray.tune.grid_search(["SET"]), network="gsc_sparse_dsnn", # network related prune_methods=[None, None, None, None], # sparse related on_perc=ray.tune.grid_search([ [perc] * 4 for perc in on_perc_levels ]), hebbian_prune_perc=None, hebbian_grow=False, weight_prune_perc=[None, None, 0.3, 0.3], ), "gsc-WeightedMag": dict( model=ray.tune.grid_search(["DSNNWeightedMag"]), network="gsc_sparse_dsnn", # network related prune_methods=[None, None, "dynamic-linear", "dynamic-linear"], # sparse related on_perc=ray.tune.grid_search([ [perc] * 4 for perc in on_perc_levels ]), hebbian_prune_perc=None, hebbian_grow=False, weight_prune_perc=[0.3, 0.3], ),



In [335]:

    
# Did any  trials failed?
df[df["epochs"]<30]["epochs"].count()









    Out[335]:





0



In [336]:

    
# Removing failed or incomplete trials
df_origin = df.copy()
df = df_origin[df_origin["epochs"]>=30]
df.shape









    Out[336]:





(20, 43)



In [337]:

    
# which ones failed?
# failed, or still ongoing?
df_origin['failed'] = df_origin["epochs"]<30
df_origin[df_origin['failed']]['epochs']









    Out[337]:





Series([], Name: epochs, dtype: int64)



In [338]:

    
# helper functions
def mean_and_std(s):
    return "{:.3f} ± {:.3f}".format(s.mean(), s.std())

def round_mean(s):
    return "{:.0f}".format(round(s.mean()))

stats = ['min', 'max', 'mean', 'std']

def agg(columns, filter=None, round=3):
    if filter is None:
        return (df.groupby(columns)
             .agg({'val_acc_max_epoch': round_mean,
                   'val_acc_max': stats,                
                   'model': ['count']})).round(round)
    else:
        return (df[filter].groupby(columns)
             .agg({'val_acc_max_epoch': round_mean,
                   'val_acc_max': stats,                
                   'model': ['count']})).round(round)

Does improved weight pruning outperforms regular SET



In [339]:

    
agg(['model'])









    Out[339]:







  
    
      
      val_acc_max_epoch
      val_acc_max
      model
    
    
      
      round_mean
      min
      max
      mean
      std
      count
    
    
      model
      
      
      
      
      
      
    
  
  
    
      DSNNWeightedMag
      83
      0.841
      0.924
      0.883
      0.037
      10
    
    
      SET
      88
      0.841
      0.922
      0.886
      0.035
      10



In [340]:

    
agg(['on_perc'])









    Out[340]:







  
    
      
      val_acc_max_epoch
      val_acc_max
      model
    
    
      
      round_mean
      min
      max
      mean
      std
      count
    
    
      on_perc
      
      
      
      
      
      
    
  
  
    
      0.02
      88
      0.841
      0.867
      0.851
      0.008
      10
    
    
      0.04
      83
      0.908
      0.924
      0.918
      0.004
      10



In [341]:

    
def model_name(row):
    
    if row['model'] == 'DSNNWeightedMag':
        return 'DSNN'

    elif row['model'] == 'SET':
        return 'SET'

    elif row['model'] == 'SparseModel':
        return 'Static'
    
    assert False, "This should cover all cases. Got {} h - {} w - {}".format(row['model'], row['hebbian_prune_perc'], row['weight_prune_perc'])

df['model2'] = df.apply(model_name, axis=1)



In [342]:

    
fltr = (df['model2'] != 'Sparse') & (df['lr_scheduler'] == "MultiStepLR")
agg(['on_perc', 'model2'], filter=fltr)









    Out[342]:







  
    
      
      
      val_acc_max_epoch
      val_acc_max
      model
    
    
      
      
      round_mean
      min
      max
      mean
      std
      count
    
    
      on_perc
      model2
      
      
      
      
      
      
    
  
  
    
      0.02
      DSNN
      83
      0.841
      0.854
      0.849
      0.006
      5
    
    
      SET
      93
      0.841
      0.867
      0.853
      0.009
      5
    
    
      0.04
      DSNN
      83
      0.908
      0.924
      0.918
      0.006
      5
    
    
      SET
      84
      0.915
      0.922
      0.918
      0.003
      5



In [343]:

    
# translate model names
rcParams['figure.figsize'] = 16, 8
# d = {
#     'DSNNWeightedMag': 'DSNN',
#     'DSNNMixedHeb': 'SET',
#     'SparseModel': 'Static',        
# }
# df_plot = df.copy()
# df_plot['model'] = df_plot['model'].apply(lambda x, i: model_name(x, i))



In [344]:

    
# sns.scatterplot(data=df_plot, x='on_perc', y='val_acc_max', hue='model')
sns.lineplot(data=df, x='on_perc', y='val_acc_max', hue='model')
plt.errorbar(x=[0.02, 0.04], y=[0.75, 0.85], yerr=[0.1, 0.01], color='k', marker='.', lw=0)









    Out[344]:





<ErrorbarContainer object of 3 artists>



In [345]:

    
rcParams['figure.figsize'] = 16, 8
filter = df['model'] != 'Static'
plt.errorbar(
    x=[0.02, 0.04],
    y=[85, 95],
    yerr=[1, 1],
    color='k',
    marker='*',
    lw=0,
    elinewidth=2,
    capsize=2,
    markersize=10,
)
sns.lineplot(data=df[filter], x='on_perc', y='val_acc_max_epoch', hue='model2')









    Out[345]:





<matplotlib.axes._subplots.AxesSubplot at 0x146c53a20>



In [346]:

    
plt.errorbar(
    x=[0.02, 0.04],
    y=[0.85, 0.95],
    yerr=[0.01, 0.01],
    color='k',
    marker='.',
    lw=0,
    elinewidth=1,
    capsize=1,
)









    Out[346]:





<ErrorbarContainer object of 3 artists>



In [ ]:



In [347]:

    
sns.lineplot(data=df, x='on_perc', y='val_acc_last', hue='model2')









    Out[347]:





<matplotlib.axes._subplots.AxesSubplot at 0x13a823c18>

	val_acc_max_epoch	val_acc_max				model
	round_mean	min	max	mean	std	count
model
DSNNWeightedMag	83	0.841	0.924	0.883	0.037	10
SET	88	0.841	0.922	0.886	0.035	10