Experiment

Run Hebbian pruning with non-binary activations.

Motivation

Attempt pruning given intuition offered up in "Memory Aware Synapses" paper:

 * The weights with higher coactivations computed as $x_i \times x_j$
 have a greater effect on the L2 norm of the layers output. Here $x_i$ and $x_j$ are
 the input and output activations respectively. 

In [1]:
from IPython.display import Markdown, display
%load_ext autoreload
%autoreload 2

In [19]:
import sys
import itertools
sys.path.append("../../")

In [3]:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import glob
import tabulate
import pprint
import click
import numpy as np
import pandas as pd
from ray.tune.commands import *
from nupic.research.frameworks.dynamic_sparse.common.browser import *

In [10]:
base = 'gsc-trials-2019-10-07'
exp_names = [
    'gsc-BaseModel',
    'gsc-Static',
    'gsc-Heb-nonbinary',
    'gsc-WeightedMag-nonbinary',
    'gsc-WeightedMag',
    'gsc-SET',
]
exps = [
    os.path.join(base, exp) for exp in exp_names
]
    
paths = [os.path.expanduser("~/nta/results/{}".format(e)) for e in exps]
for p in paths:
    print(os.path.exists(p), p)
df = load_many(paths)


True /Users/mcaporale/nta/results/gsc-trials-2019-10-07/gsc-BaseModel
True /Users/mcaporale/nta/results/gsc-trials-2019-10-07/gsc-Static
True /Users/mcaporale/nta/results/gsc-trials-2019-10-07/gsc-Heb-nonbinary
True /Users/mcaporale/nta/results/gsc-trials-2019-10-07/gsc-WeightedMag-nonbinary
True /Users/mcaporale/nta/results/gsc-trials-2019-10-07/gsc-WeightedMag
True /Users/mcaporale/nta/results/gsc-trials-2019-10-07/gsc-SET
gsc-BaseModel
gsc-Static
gsc-Heb-nonbinary
gsc-WeightedMag-nonbinary
gsc-WeightedMag
gsc-SET

In [11]:
# remove nans where appropriate
df['hebbian_prune_perc'] = df['hebbian_prune_perc'].replace(np.nan, 0.0, regex=True)
df['weight_prune_perc'] = df['weight_prune_perc'].replace(np.nan, 0.0, regex=True)

# distill certain values 
df['on_perc'] = df['on_perc'].replace('None-None-0.1-None', 0.1, regex=True)
df['on_perc'] = df['on_perc'].replace('None-None-0.4-None', 0.4, regex=True)
df['on_perc'] = df['on_perc'].replace('None-None-0.02-None', 0.02, regex=True)
df['prune_methods'] = df['prune_methods'].replace('None-None-dynamic-linear-None', 'dynamic-linear', regex=True)

In [186]:
# def model_name(row):
#     col = 'Experiment Name'
    
#     for exp in exp_names:
#         if exp in row[col]:
#             return exp

# #     if row[col] == 'DSNNWeightedMag':
# #         return 'DSNN-WM'

# #     elif row[col] == 'DSNNMixedHeb':
# #         if row['hebbian_prune_perc'] == 0.3:
# #            return 'SET'

# #         elif row['weight_prune_perc'] == 0.3:
# #            return 'DSNN-Heb'

# #     elif row[col] == 'SparseModel':
# #         return 'Static'
    
#     assert False, "This should cover all cases. Got {}".format(row[col])

# df['model2'] = df.apply(model_name, axis=1)

In [169]:
df.iloc[34]


Out[169]:
Experiment Name             28_hebbian_grow=True,model=DSNNMixedHeb,moving...
train_acc_max                                                         0.87179
train_acc_max_epoch                                                        25
train_acc_min                                                        0.692462
train_acc_min_epoch                                                         0
train_acc_median                                                     0.839713
train_acc_last                                                       0.864125
val_acc_max                                                          0.926624
val_acc_max_epoch                                                          29
val_acc_min                                                          0.126303
val_acc_min_epoch                                                           5
val_acc_median                                                       0.844828
val_acc_last                                                         0.926624
val_acc_all                 0     0.899358
1     0.833601
2     0.400561
3...
epochs                                                                     30
experiment_file_name        /Users/mcaporale/nta/results/gsc-trials-2019-1...
trial_time                                                            7.68563
mean_epoch_time                                                      0.256188
batch_size_test                                                          1000
batch_size_train                                                           16
data_dir                                                   ~/nta/datasets/gsc
dataset_name                                                  PreprocessedGSC
debug_sparse                                                             True
debug_weights                                                            True
device                                                                   cuda
learning_rate                                                            0.01
lr_gamma                                                                  0.9
lr_scheduler                                                           StepLR
lr_step_size                                                                1
model                                                            DSNNMixedHeb
momentum                                                                    0
network                                                                GSCHeb
on_perc                                                                  0.02
optim_alg                                                                 SGD
test_noise                                                              False
weight_decay                                                             0.01
hebbian_grow                                                             True
hebbian_prune_perc                                                        0.3
moving_average_alpha                                                        1
prune_methods                                                  dynamic-linear
reset_coactivations                                                      True
use_binary_coactivations                                                False
weight_prune_perc                                                           0
Name: 34, dtype: object

In [13]:
df.groupby('experiment_base_path')['experiment_base_path'].count()


Out[13]:
experiment_base_path
gsc-BaseModel                 6
gsc-Heb-nonbinary            36
gsc-SET                       6
gsc-Static                    6
gsc-WeightedMag               6
gsc-WeightedMag-nonbinary    12
Name: experiment_base_path, dtype: int64

In [170]:
# Did anything fail?
df[df["epochs"] < 30]["epochs"].count()


Out[170]:
0

In [15]:
# helper functions
def mean_and_std(s):
    return "{:.3f} ± {:.3f}".format(s.mean(), s.std())

def round_mean(s):
    return "{:.0f}".format(round(s.mean()))

stats = ['min', 'max', 'mean', 'std']

def agg(columns, filter=None, round=3):
    if filter is None:
        return (df.groupby(columns)
             .agg({'val_acc_max_epoch': round_mean,
                   'val_acc_max': stats,                
                   'model': ['count']})).round(round)
    else:
        return (df[filter].groupby(columns)
             .agg({'val_acc_max_epoch': round_mean,
                   'val_acc_max': stats,                
                   'model': ['count']})).round(round)

In [143]:
type(np.nan)


Out[143]:
float

In [150]:
df['on_perc'][0] is nan


Out[150]:
False

Dense Model


In [16]:
fltr = (df['experiment_base_path'] == 'gsc-BaseModel')
agg(['model'], fltr)


Out[16]:
val_acc_max_epoch val_acc_max model
round_mean min max mean std count
model
BaseModel 26 0.964 0.971 0.966 0.003 6

Static Sparse


In [17]:
# 2% sparse
fltr = (df['experiment_base_path'] == 'gsc-Static')
agg(['model'], fltr)


Out[17]:
val_acc_max_epoch val_acc_max model
round_mean min max mean std count
model
SparseModel 25 0.958 0.963 0.961 0.002 6

Weighted Magnitude


In [20]:
# 2% sparse
# 2% sparse 
combos = {
    'experiment_base_path': ['gsc-WeightedMag', 'gsc-WeightedMag-nonbinary'],
    'hebbian_grow': [True, False],
}
combos = [[(k, v_i) for v_i in v] for k, v in combos.items()]
combos = list(itertools.product(*combos))

for c in combos:
    fltr = None
    summary = []
    for restraint in c:
        
        rname = restraint[0]
        rcond = restraint[1]
        
        summary.append("{}={} ".format(rname, rcond))
        
        new_fltr = df[rname] == rcond
        if fltr is not None:
            fltr = fltr & new_fltr
        else:
            fltr = new_fltr
            
    summary = Markdown("### " + " / ".join(summary))
    display(summary)
    display(agg(['experiment_base_path'], fltr))
    print('\n\n\n\n')


experiment_base_path=gsc-WeightedMag / hebbian_grow=True

val_acc_max_epoch val_acc_max model
round_mean min max mean std count
experiment_base_path




experiment_base_path=gsc-WeightedMag / hebbian_grow=False

val_acc_max_epoch val_acc_max model
round_mean min max mean std count
experiment_base_path
gsc-WeightedMag 27 0.96 0.965 0.963 0.002 6




experiment_base_path=gsc-WeightedMag-nonbinary / hebbian_grow=True

val_acc_max_epoch val_acc_max model
round_mean min max mean std count
experiment_base_path
gsc-WeightedMag-nonbinary 23 0.961 0.964 0.962 0.001 6




experiment_base_path=gsc-WeightedMag-nonbinary / hebbian_grow=False

val_acc_max_epoch val_acc_max model
round_mean min max mean std count
experiment_base_path
gsc-WeightedMag-nonbinary 26 0.962 0.966 0.964 0.002 6




SET


In [21]:
# 2% sparse 
fltr = (df['experiment_base_path'] == 'gsc-SET')
display(agg(['model'], fltr))


val_acc_max_epoch val_acc_max model
round_mean min max mean std count
model
DSNNMixedHeb 20 0.958 0.965 0.962 0.002 6

Hebbien


In [236]:
# 2% sparse 
combos = {
    'hebbian_grow': [True, False],
    'moving_average_alpha': [0.6, 0.8, 1.0],
    'reset_coactivations': [True, False],
}
combos = [[(k, v_i) for v_i in v] for k, v in combos.items()]
combos = list(itertools.product(*combos))

for c in combos:
    fltr = None
    summary = []
    for restraint in c:
        
        rname = restraint[0]
        rcond = restraint[1]
        
        summary.append("{}={} ".format(rname, rcond))
        
        new_fltr = df[rname] == rcond
        if fltr is not None:
            fltr = fltr & new_fltr
        else:
            fltr = new_fltr
            
    summary = Markdown("### " + " / ".join(summary))
    display(summary)
    display(agg(['experiment_base_path'], fltr))
    print('\n\n\n\n')


hebbian_grow=True / moving_average_alpha=0.6 / reset_coactivations=True

val_acc_max_epoch val_acc_max model
round_mean min max mean std count
experiment_base_path
gsc-Heb-nonbinary 24 0.955 0.961 0.958 0.003 3




hebbian_grow=True / moving_average_alpha=0.6 / reset_coactivations=False

val_acc_max_epoch val_acc_max model
round_mean min max mean std count
experiment_base_path
gsc-Heb-nonbinary 23 0.956 0.957 0.957 0.001 3




hebbian_grow=True / moving_average_alpha=0.8 / reset_coactivations=True

val_acc_max_epoch val_acc_max model
round_mean min max mean std count
experiment_base_path
gsc-Heb-nonbinary 26 0.957 0.957 0.957 0.0 3




hebbian_grow=True / moving_average_alpha=0.8 / reset_coactivations=False

val_acc_max_epoch val_acc_max model
round_mean min max mean std count
experiment_base_path
gsc-Heb-nonbinary 20 0.956 0.959 0.958 0.002 3




hebbian_grow=True / moving_average_alpha=1.0 / reset_coactivations=True

val_acc_max_epoch val_acc_max model
round_mean min max mean std count
experiment_base_path
gsc-Heb-nonbinary 24 0.927 0.93 0.928 0.002 3




hebbian_grow=True / moving_average_alpha=1.0 / reset_coactivations=False

val_acc_max_epoch val_acc_max model
round_mean min max mean std count
experiment_base_path
gsc-Heb-nonbinary 28 0.913 0.931 0.924 0.01 3




hebbian_grow=False / moving_average_alpha=0.6 / reset_coactivations=True

val_acc_max_epoch val_acc_max model
round_mean min max mean std count
experiment_base_path
gsc-Heb-nonbinary 21 0.957 0.959 0.958 0.001 3




hebbian_grow=False / moving_average_alpha=0.6 / reset_coactivations=False

val_acc_max_epoch val_acc_max model
round_mean min max mean std count
experiment_base_path
gsc-Heb-nonbinary 24 0.959 0.962 0.961 0.002 3




hebbian_grow=False / moving_average_alpha=0.8 / reset_coactivations=True

val_acc_max_epoch val_acc_max model
round_mean min max mean std count
experiment_base_path
gsc-Heb-nonbinary 20 0.959 0.961 0.96 0.001 3




hebbian_grow=False / moving_average_alpha=0.8 / reset_coactivations=False

val_acc_max_epoch val_acc_max model
round_mean min max mean std count
experiment_base_path
gsc-Heb-nonbinary 25 0.957 0.962 0.96 0.002 3




hebbian_grow=False / moving_average_alpha=1.0 / reset_coactivations=True

val_acc_max_epoch val_acc_max model
round_mean min max mean std count
experiment_base_path
gsc-Heb-nonbinary 26 0.947 0.949 0.948 0.001 3




hebbian_grow=False / moving_average_alpha=1.0 / reset_coactivations=False

val_acc_max_epoch val_acc_max model
round_mean min max mean std count
experiment_base_path
gsc-Heb-nonbinary 24 0.946 0.949 0.947 0.001 3





In [22]:
d = {'b':4}
'b' in d


Out[22]:
True

In [ ]: