In [15]:
%load_ext autoreload
%autoreload 2
In [16]:
base_exp_config = dict(
device="cuda",
# ----- dataset related ----
dataset_name="PreprocessedGSC",
data_dir=os.path.expanduser("~/nta/datasets/gsc"),
train_batches_per_epoch=5121,
# batch_size_train=(4, 16),
batch_size_train=16,
batch_size_test=20, # required to fit the GPU
# ----- network related ----
network="GSCHeb",
percent_on_k_winner=[0.095, 0.125, 0.067],
k_inference_factor=1.5,
boost_strength=[1.5, 1.5, 1.5],
boost_strength_factor=[0.9, 0.9, 0.9],
hidden_neurons_conv=[64, 64],
hidden_neurons_fc=1500,
bias=True,
dropout=False,
batch_norm=True,
# ----- model related ----
model="BaseModel",
optim_alg="SGD",
momentum=0,
learning_rate=0.01,
weight_decay=0.01,
lr_scheduler="StepLR",
lr_gamma=0.9,
on_perc=[1, 1, 0.1, 1],
hebbian_prune_perc=None,
hebbian_grow=False,
weight_prune_perc=0.3,
pruning_early_stop=None, # 2
# additional validation
test_noise=True,
# debugging
debug_weights=True,
debug_sparse=True,
)
In [17]:
from nupic.research.frameworks.dynamic_sparse.networks import GSCHeb
In [20]:
net = GSCHeb(config=base_exp_config)
In [21]:
print(net)
In [11]:
from nupic.research.frameworks.dynamic_sparse.networks import GSCHeb
from torchsummary import summary
import torch
import os
In [12]:
# net = GSCHeb()
net = GSCHeb()
In [13]:
summary(net, input_size=(1,32,32))
In [26]:
net.train()
for _ in range(1000):
net(torch.rand(2,1,32,32))
In [2]:
num_dense = sum([784*100, 100*100, 100*100, 100*10])
num_sparse = num_dense * .1
num_dense, num_sparse
Out[2]:
In [3]:
# need to sum bias
In [8]:
import sys
sys.path.append("../../")
from nupic.research.frameworks.dynamic_sparse.models import BaseModel
from nupic.research.frameworks.dynamic_sparse.networks import MLP
import torch
import numpy as np
# load a regular network
network = MLP()
# load a regular base model for training
model = BaseModel(network=network)
model.setup()
In [9]:
total = 0
for m in network.classifier.modules():
if isinstance(m, torch.nn.Linear):
num_weights = np.prod(m.weight.shape)
num_bias = np.prod(m.bias.shape)
print(num_weights, num_bias)
total = total + num_weights + num_bias
print("\n", total)
num_dense = total
In [10]:
total = 0
for m in network.classifier.modules():
if isinstance(m, torch.nn.Linear):
num_weights = int(np.prod(m.weight.shape)*.2)
num_bias = np.prod(m.bias.shape)
print(num_weights, num_bias)
total = total + num_weights + num_bias
print("\n", total)
num_sparse = total
In [11]:
total = 0
for m in network.classifier.modules():
if isinstance(m, torch.nn.Linear):
num_weights = int(np.prod(m.weight.shape)*.1)
num_bias = np.prod(m.bias.shape)
print(num_weights, num_bias)
total = total + num_weights + num_bias
print("\n", total)
num_sparse = total
In [12]:
total = 0
for m in network.classifier.modules():
if isinstance(m, torch.nn.Linear):
num_weights = int(np.prod(m.weight.shape)*.05)
num_bias = np.prod(m.bias.shape)
print(num_weights, num_bias)
total = total + num_weights + num_bias
print("\n", total)
num_sparse = total
In [13]:
hs = [12,16,23]
sum([784*hs[0], hs[0]*hs[1], hs[1]*hs[2], hs[2]*10]) + sum(hs)
Out[13]:
In [1]:
from nupic.research.frameworks.dynamic_sparse.networks import GSCHeb, GSCHebSmall
In [2]:
network = GSCHeb()
In [3]:
from torchsummary import summary
In [4]:
summary(network, input_size=(1,32,32))
In [5]:
# Total params = 1,717,140
# 4% of that =
1717140 * 0.04
Out[5]:
In [6]:
64/25
Out[6]:
In [ ]:
In [7]:
network= GSCHebSmall()
summary(network, input_size=(1,32,32))
In [ ]: