In [67]:
%load_ext autoreload
%autoreload 2
import matplotlib.pyplot as plt
from matplotlib import rcParams
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
rcParams['figure.figsize'] = (12,6)
import numpy as np
import pandas as pd
import seaborn as sns
import os
import torch
from torch import nn
from torchvision import models
from torchsummary import summary
from ray import tune
The autoreload extension is already loaded. To reload it, use:
%reload_ext autoreload
In [135]:
exp_config = dict(
#### model related
device="cuda",
# network=tune.grid_search(["vgg19_bn_kw", "vgg19_bn"]),
network="VGG19",
num_classes=10,
model=tune.grid_search(["BaseModel", "SparseModel", "SET", "DSNN"]),
# model=tune.grid_search(["SET", "DSNN", "DSNN_Flip", "DSNN_Correct"]),
dataset_name="CIFAR10",
augment_images=True,
stats_mean=(0.4914, 0.4822, 0.4465),
stats_std=(0.2023, 0.1994, 0.2010),
data_dir="~/nta/datasets",
#### optimizer related
optim_alg="SGD",
momentum=0.9,
learning_rate=0.01,
lr_scheduler="MultiStepLR",
lr_milestones=[250,290],
lr_gamma=0.10,
debug_weights=True,
#### sparse related
epsilon=100,
start_sparse=1,
end_sparse=None,
debug_sparse=True,
flip=False,
weight_prune_perc=0.3,
grad_prune_perc=0,
percent_on=0.3,
boost_strength=1.4,
boost_strength_factor=0.7,
test_noise=True,
noise_level=0.1,
kwinners=True # moved to a parameter
)
In [141]:
from networks import VGG19
model = VGG19(exp_config)
<bound method VGG19._kwinners of VGG19()>
<class 'nupic.torch.modules.k_winners.KWinners2d'>
KWinners2d(channels=10, n=0, percent_on=0.3, boost_strength=1.4, duty_cycle_period=1000)
In [ ]:
In [33]:
model = models.vgg19_bn()
In [7]:
model.features
Out[7]:
Sequential(
(0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(2): ReLU(inplace)
(3): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(4): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(5): ReLU(inplace)
(6): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
(7): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(8): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(9): ReLU(inplace)
(10): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(11): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(12): ReLU(inplace)
(13): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
(14): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(15): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(16): ReLU(inplace)
(17): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(18): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(19): ReLU(inplace)
(20): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(21): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(22): ReLU(inplace)
(23): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(24): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(25): ReLU(inplace)
(26): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
(27): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(28): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(29): ReLU(inplace)
(30): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(31): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(32): ReLU(inplace)
(33): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(34): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(35): ReLU(inplace)
(36): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(37): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(38): ReLU(inplace)
(39): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
(40): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(41): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(42): ReLU(inplace)
(43): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(44): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(45): ReLU(inplace)
(46): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(47): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(48): ReLU(inplace)
(49): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(50): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(51): ReLU(inplace)
(52): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
)
In [61]:
model = models.vgg19_bn()
config = {
'percent_on': 0.3,
'boost_strength': 1.4,
'boost_strength_factor': 0.7
}
new_features = []
for layer in model.features:
# remove max pooling
if isinstance(layer, nn.MaxPool2d):
pass
elif isinstance(layer, nn.Conv2d):
new_features.append(layer)
last_conv_out_channels = layer.out_channels
# switch ReLU to kWinners2d
elif isinstance(layer, nn.ReLU):
new_features.append(KWinners2d(
channels=last_conv_out_channels,
percent_on=config['percent_on'],
boost_strength=config['boost_strength'],
boost_strength_factor=config['boost_strength_factor']))
# otherwise add it as normal
else:
new_features.append(layer)
model.features = nn.Sequential(*new_features)
In [62]:
model
Out[62]:
VGG(
(features): Sequential(
(0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(2): KWinners2d(channels=64, n=0, percent_on=0.3, boost_strength=1.4, duty_cycle_period=1000)
(3): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(4): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(5): KWinners2d(channels=64, n=0, percent_on=0.3, boost_strength=1.4, duty_cycle_period=1000)
(6): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(7): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(8): KWinners2d(channels=128, n=0, percent_on=0.3, boost_strength=1.4, duty_cycle_period=1000)
(9): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(10): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(11): KWinners2d(channels=128, n=0, percent_on=0.3, boost_strength=1.4, duty_cycle_period=1000)
(12): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(13): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(14): KWinners2d(channels=256, n=0, percent_on=0.3, boost_strength=1.4, duty_cycle_period=1000)
(15): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(16): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(17): KWinners2d(channels=256, n=0, percent_on=0.3, boost_strength=1.4, duty_cycle_period=1000)
(18): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(19): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(20): KWinners2d(channels=256, n=0, percent_on=0.3, boost_strength=1.4, duty_cycle_period=1000)
(21): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(22): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(23): KWinners2d(channels=256, n=0, percent_on=0.3, boost_strength=1.4, duty_cycle_period=1000)
(24): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(25): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(26): KWinners2d(channels=512, n=0, percent_on=0.3, boost_strength=1.4, duty_cycle_period=1000)
(27): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(28): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(29): KWinners2d(channels=512, n=0, percent_on=0.3, boost_strength=1.4, duty_cycle_period=1000)
(30): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(31): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(32): KWinners2d(channels=512, n=0, percent_on=0.3, boost_strength=1.4, duty_cycle_period=1000)
(33): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(34): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(35): KWinners2d(channels=512, n=0, percent_on=0.3, boost_strength=1.4, duty_cycle_period=1000)
(36): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(37): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(38): KWinners2d(channels=512, n=0, percent_on=0.3, boost_strength=1.4, duty_cycle_period=1000)
(39): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(40): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(41): KWinners2d(channels=512, n=0, percent_on=0.3, boost_strength=1.4, duty_cycle_period=1000)
(42): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(43): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(44): KWinners2d(channels=512, n=0, percent_on=0.3, boost_strength=1.4, duty_cycle_period=1000)
(45): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(46): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(47): KWinners2d(channels=512, n=0, percent_on=0.3, boost_strength=1.4, duty_cycle_period=1000)
)
(avgpool): AdaptiveAvgPool2d(output_size=(7, 7))
(classifier): Sequential(
(0): Linear(in_features=25088, out_features=4096, bias=True)
(1): ReLU(inplace)
(2): Dropout(p=0.5)
(3): Linear(in_features=4096, out_features=4096, bias=True)
(4): ReLU(inplace)
(5): Dropout(p=0.5)
(6): Linear(in_features=4096, out_features=1000, bias=True)
)
)
In [63]:
summary(model, input_size=(3,32,32))
----------------------------------------------------------------
Layer (type) Output Shape Param #
================================================================
Conv2d-1 [-1, 64, 32, 32] 1,792
BatchNorm2d-2 [-1, 64, 32, 32] 128
KWinners2d-3 [-1, 64, 32, 32] 0
Conv2d-4 [-1, 64, 32, 32] 36,928
BatchNorm2d-5 [-1, 64, 32, 32] 128
KWinners2d-6 [-1, 64, 32, 32] 0
Conv2d-7 [-1, 128, 32, 32] 73,856
BatchNorm2d-8 [-1, 128, 32, 32] 256
KWinners2d-9 [-1, 128, 32, 32] 0
Conv2d-10 [-1, 128, 32, 32] 147,584
BatchNorm2d-11 [-1, 128, 32, 32] 256
KWinners2d-12 [-1, 128, 32, 32] 0
Conv2d-13 [-1, 256, 32, 32] 295,168
BatchNorm2d-14 [-1, 256, 32, 32] 512
KWinners2d-15 [-1, 256, 32, 32] 0
Conv2d-16 [-1, 256, 32, 32] 590,080
BatchNorm2d-17 [-1, 256, 32, 32] 512
KWinners2d-18 [-1, 256, 32, 32] 0
Conv2d-19 [-1, 256, 32, 32] 590,080
BatchNorm2d-20 [-1, 256, 32, 32] 512
KWinners2d-21 [-1, 256, 32, 32] 0
Conv2d-22 [-1, 256, 32, 32] 590,080
BatchNorm2d-23 [-1, 256, 32, 32] 512
KWinners2d-24 [-1, 256, 32, 32] 0
Conv2d-25 [-1, 512, 32, 32] 1,180,160
BatchNorm2d-26 [-1, 512, 32, 32] 1,024
KWinners2d-27 [-1, 512, 32, 32] 0
Conv2d-28 [-1, 512, 32, 32] 2,359,808
BatchNorm2d-29 [-1, 512, 32, 32] 1,024
KWinners2d-30 [-1, 512, 32, 32] 0
Conv2d-31 [-1, 512, 32, 32] 2,359,808
BatchNorm2d-32 [-1, 512, 32, 32] 1,024
KWinners2d-33 [-1, 512, 32, 32] 0
Conv2d-34 [-1, 512, 32, 32] 2,359,808
BatchNorm2d-35 [-1, 512, 32, 32] 1,024
KWinners2d-36 [-1, 512, 32, 32] 0
Conv2d-37 [-1, 512, 32, 32] 2,359,808
BatchNorm2d-38 [-1, 512, 32, 32] 1,024
KWinners2d-39 [-1, 512, 32, 32] 0
Conv2d-40 [-1, 512, 32, 32] 2,359,808
BatchNorm2d-41 [-1, 512, 32, 32] 1,024
KWinners2d-42 [-1, 512, 32, 32] 0
Conv2d-43 [-1, 512, 32, 32] 2,359,808
BatchNorm2d-44 [-1, 512, 32, 32] 1,024
KWinners2d-45 [-1, 512, 32, 32] 0
Conv2d-46 [-1, 512, 32, 32] 2,359,808
BatchNorm2d-47 [-1, 512, 32, 32] 1,024
KWinners2d-48 [-1, 512, 32, 32] 0
AdaptiveAvgPool2d-49 [-1, 512, 7, 7] 0
Linear-50 [-1, 4096] 102,764,544
ReLU-51 [-1, 4096] 0
Dropout-52 [-1, 4096] 0
Linear-53 [-1, 4096] 16,781,312
ReLU-54 [-1, 4096] 0
Dropout-55 [-1, 4096] 0
Linear-56 [-1, 1000] 4,097,000
================================================================
Total params: 143,678,248
Trainable params: 143,678,248
Non-trainable params: 0
----------------------------------------------------------------
Input size (MB): 0.01
Forward/backward pass size (MB): 129.39
Params size (MB): 548.09
Estimated Total Size (MB): 677.49
----------------------------------------------------------------
In [42]:
from nupic.torch.modules import KWinners2d
new_features = []
for layer in model.features:
if isinstance(layer, nn.MaxPool2d):
pass
elif isinstance(layer, nn.ReLU):
new_features.append(KWinners2d(0, percent_on=0.3))
else:
new_features.append(layer)
In [46]:
model.features = nn.Sequential(*new_features)
summary(model, input_size=(3,32,32))
---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
<ipython-input-46-c069c7653900> in <module>
1 model.features = nn.Sequential(*new_features)
----> 2 summary(model, input_size=(3,32,32))
~/miniconda3/envs/numenta/lib/python3.7/site-packages/torchsummary/torchsummary.py in summary(model, input_size, batch_size, device)
70 # make a forward pass
71 # print(x.shape)
---> 72 model(*x)
73
74 # remove these hooks
~/miniconda3/envs/numenta/lib/python3.7/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
491 result = self._slow_forward(*input, **kwargs)
492 else:
--> 493 result = self.forward(*input, **kwargs)
494 for hook in self._forward_hooks.values():
495 hook_result = hook(self, input, result)
~/miniconda3/envs/numenta/lib/python3.7/site-packages/torchvision/models/vgg.py in forward(self, x)
40
41 def forward(self, x):
---> 42 x = self.features(x)
43 x = self.avgpool(x)
44 x = x.view(x.size(0), -1)
~/miniconda3/envs/numenta/lib/python3.7/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
491 result = self._slow_forward(*input, **kwargs)
492 else:
--> 493 result = self.forward(*input, **kwargs)
494 for hook in self._forward_hooks.values():
495 hook_result = hook(self, input, result)
~/miniconda3/envs/numenta/lib/python3.7/site-packages/torch/nn/modules/container.py in forward(self, input)
90 def forward(self, input):
91 for module in self._modules.values():
---> 92 input = module(input)
93 return input
94
~/miniconda3/envs/numenta/lib/python3.7/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
491 result = self._slow_forward(*input, **kwargs)
492 else:
--> 493 result = self.forward(*input, **kwargs)
494 for hook in self._forward_hooks.values():
495 hook_result = hook(self, input, result)
~/nta/nupic.torch/nupic/torch/modules/k_winners.py in forward(self, x)
267
268 if self.training:
--> 269 x = F.KWinners2d.apply(x, self.duty_cycle, self.k, self.boost_strength)
270 self.update_duty_cycle(x)
271
~/nta/nupic.torch/nupic/torch/functions/k_winners.py in forward(ctx, x, duty_cycles, k, boost_strength)
165 target_density = float(k) / (x.shape[1] * x.shape[2] * x.shape[3])
166 boost_factors = torch.exp((target_density - duty_cycles) * boost_strength)
--> 167 boosted = x.detach() * boost_factors
168 else:
169 boosted = x.detach()
RuntimeError: The size of tensor a (64) must match the size of tensor b (0) at non-singleton dimension 1
In [44]:
model
Out[44]:
VGG(
(features): Sequential(
(0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(2): KWinners2d(channels=0, n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(3): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(4): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(5): KWinners2d(channels=0, n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(6): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(7): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(8): KWinners2d(channels=0, n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(9): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(10): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(11): KWinners2d(channels=0, n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(12): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(13): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(14): KWinners2d(channels=0, n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(15): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(16): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(17): KWinners2d(channels=0, n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(18): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(19): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(20): KWinners2d(channels=0, n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(21): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(22): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(23): KWinners2d(channels=0, n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(24): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(25): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(26): KWinners2d(channels=0, n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(27): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(28): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(29): KWinners2d(channels=0, n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(30): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(31): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(32): KWinners2d(channels=0, n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(33): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(34): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(35): KWinners2d(channels=0, n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(36): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(37): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(38): KWinners2d(channels=0, n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(39): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(40): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(41): KWinners2d(channels=0, n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(42): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(43): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(44): KWinners2d(channels=0, n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(45): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(46): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(47): KWinners2d(channels=0, n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
)
(avgpool): AdaptiveAvgPool2d(output_size=(7, 7))
(classifier): Sequential(
(0): Linear(in_features=25088, out_features=4096, bias=True)
(1): ReLU(inplace)
(2): Dropout(p=0.5)
(3): Linear(in_features=4096, out_features=4096, bias=True)
(4): ReLU(inplace)
(5): Dropout(p=0.5)
(6): Linear(in_features=4096, out_features=1000, bias=True)
)
)
In [31]:
from nupic.torch.modules import KWinners
for idx, layer in enumerate(model.features):
if isinstance(layer, nn.ReLU):
model.features[idx] = KWinners(0, 0.3)
elif is
In [30]:
model
Out[30]:
VGG(
(features): Sequential(
(0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(2): KWinners(n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(3): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(4): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(5): KWinners(n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(6): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
(7): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(8): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(9): KWinners(n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(10): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(11): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(12): KWinners(n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(13): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
(14): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(15): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(16): KWinners(n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(17): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(18): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(19): KWinners(n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(20): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(21): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(22): KWinners(n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(23): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(24): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(25): KWinners(n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(26): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
(27): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(28): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(29): KWinners(n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(30): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(31): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(32): KWinners(n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(33): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(34): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(35): KWinners(n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(36): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(37): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(38): KWinners(n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(39): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
(40): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(41): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(42): KWinners(n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(43): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(44): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(45): KWinners(n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(46): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(47): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(48): KWinners(n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(49): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(50): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(51): KWinners(n=0, percent_on=0.3, boost_strength=1.0, duty_cycle_period=1000)
(52): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
)
(avgpool): AdaptiveAvgPool2d(output_size=(7, 7))
(classifier): Sequential(
(0): Linear(in_features=25088, out_features=4096, bias=True)
(1): ReLU(inplace)
(2): Dropout(p=0.5)
(3): Linear(in_features=4096, out_features=4096, bias=True)
(4): ReLU(inplace)
(5): Dropout(p=0.5)
(6): Linear(in_features=4096, out_features=1000, bias=True)
)
)
In [10]:
dir(m)
Out[10]:
['__call__',
'__class__',
'__constants__',
'__delattr__',
'__dict__',
'__dir__',
'__doc__',
'__eq__',
'__format__',
'__ge__',
'__getattr__',
'__getattribute__',
'__gt__',
'__hash__',
'__init__',
'__init_subclass__',
'__le__',
'__lt__',
'__module__',
'__ne__',
'__new__',
'__reduce__',
'__reduce_ex__',
'__repr__',
'__setattr__',
'__setstate__',
'__sizeof__',
'__str__',
'__subclasshook__',
'__weakref__',
'_apply',
'_backend',
'_backward_hooks',
'_buffers',
'_forward_hooks',
'_forward_pre_hooks',
'_get_name',
'_load_from_state_dict',
'_load_state_dict_pre_hooks',
'_modules',
'_named_members',
'_parameters',
'_register_load_state_dict_pre_hook',
'_register_state_dict_hook',
'_slow_forward',
'_state_dict_hooks',
'_tracing_name',
'_version',
'add_module',
'apply',
'bias',
'buffers',
'children',
'cpu',
'cuda',
'double',
'dump_patches',
'eval',
'extra_repr',
'float',
'forward',
'half',
'in_features',
'load_state_dict',
'modules',
'named_buffers',
'named_children',
'named_modules',
'named_parameters',
'out_features',
'parameters',
'register_backward_hook',
'register_buffer',
'register_forward_hook',
'register_forward_pre_hook',
'register_parameter',
'reset_parameters',
'share_memory',
'state_dict',
'to',
'train',
'training',
'type',
'weight',
'zero_grad']
In [ ]:
class cauchy_activation(nn.Module):
def __init__(self):
super(cauchy_activation, self).__init__()
def activation(self, inp):
return pdf_cauchy_distribution(self.inp)
Content source: numenta/nupic.research
Similar notebooks: