In [1]:

    

%load_ext autoreload
%autoreload 2


    

In [41]:

    

import torch
import torch.nn.functional as F
import torch.nn as nn


    

In [3]:

    

w1 = torch.randn((3,4))
w2 = torch.randn((4,5))


    

In [4]:

    

u0 = torch.randn((1,3))
u0_sample1 = torch.randn((1,4))
u0_sample2 = torch.randn((1,4))
u0_sample3 = torch.randn((1,4))


    

In [5]:

    

u1 = u0 @ w1
a1 = F.relu(u1)

u2 = a1 @ w2
a2 = F.relu(u2)


    

In [6]:

    

u0, u1, u2


    

    
Out[6]:





(tensor([[ 1.5046, -0.9168, -0.1071]]),
 tensor([[ 0.7525, -2.2559,  1.2341, -1.9694]]),
 tensor([[-1.7027,  0.1315,  0.5429, -1.1076, -2.6565]]))

In [7]:

    

a1, a2
bin_a1 = a1.bool()
bin_a2 = a2.bool()
bin_a1, bin_a2

torch.ger(bin_a1.view(-1).int(), bin_a2.view(-1).int())


    

    
Out[7]:





tensor([[0, 1, 1, 0, 0],
        [0, 0, 0, 0, 0],
        [0, 1, 1, 0, 0],
        [0, 0, 0, 0, 0]], dtype=torch.int32)

In [8]:

    

bin_a1, bin_a2


    

    
Out[8]:





(tensor([[ True, False,  True, False]]),
 tensor([[False,  True,  True, False, False]]))

In [9]:

    

l0 @ w0


    

    




---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-9-dda50510556a> in <module>
----> 1 l0 @ w0

NameError: name 'l0' is not defined

In [ ]:

    

for i in range(0):
    print('test')


    

In [10]:

    

[1] + [2]


    

    
Out[10]:





[1, 2]

In [ ]:

    




    

In [ ]:

    




    

In [ ]:

    




    

In [11]:

    

l1 = torch.rand(4,4) - 0.5
l2 = torch.rand(4,4) - 0.5


    

In [12]:

    

l1, l2


    

    
Out[12]:





(tensor([[-0.4650,  0.4306,  0.3862, -0.3515],
         [ 0.3349, -0.3047,  0.4430,  0.1848],
         [-0.2227, -0.1486,  0.4093,  0.3325],
         [ 0.4175, -0.2552, -0.0873, -0.2341]]),
 tensor([[-0.4936,  0.3309, -0.3928, -0.4395],
         [ 0.3285, -0.4350,  0.0719, -0.4168],
         [ 0.3718, -0.1376, -0.1727,  0.3263],
         [ 0.2117,  0.0024,  0.0549,  0.1020]]))

In [13]:

    

F.relu(l1)


    

    
Out[13]:





tensor([[0.0000, 0.4306, 0.3862, 0.0000],
        [0.3349, 0.0000, 0.4430, 0.1848],
        [0.0000, 0.0000, 0.4093, 0.3325],
        [0.4175, 0.0000, 0.0000, 0.0000]])

In [14]:

    

i = torch.randn((1,4))
i


    

    
Out[14]:





tensor([[ 0.9678,  0.5911,  1.7775, -1.3768]])

In [15]:

    

h1 = i @ l1
h1


    

    
Out[15]:





tensor([[-1.2227,  0.3239,  1.4834,  0.6823]])

In [16]:

    

h2 = h1 @ l2
h2


    

    
Out[16]:





tensor([[ 1.4060, -0.7480,  0.2848,  0.9559]])

In [17]:

    

from nupic.research.frameworks.dynamic_sparse import networks
from torchsummary import summary


    

In [18]:

    

# net = networks.MLP()
# net = networks.MLPHeb(config=dict(
#   input_size=784,
#   num_classes=10,
#   hidden_sizes=[100,100,100],
#   use_kwinners=False,
# ))


    

In [38]:

    

net = networks.MLPHeb(config=dict(
  input_size=3,
  num_classes=2,
  hidden_sizes=[4, 5],
  use_kwinners=False,
))


    

In [39]:

    

summary(net, input_size=(1,3))


    

    




----------------------------------------------------------------
        Layer (type)               Output Shape         Param #
================================================================
            Linear-1                    [-1, 4]              16
              ReLU-2                    [-1, 4]               0
            Linear-3                    [-1, 5]              25
              ReLU-4                    [-1, 5]               0
            Linear-5                    [-1, 2]              12
================================================================
Total params: 53
Trainable params: 53
Non-trainable params: 0
----------------------------------------------------------------
Input size (MB): 0.00
Forward/backward pass size (MB): 0.00
Params size (MB): 0.00
Estimated Total Size (MB): 0.00
----------------------------------------------------------------

In [43]:

    

for m in net.modules():
    if isinstance(m, nn.Linear):
        print(m.weight.shape)


    

    




torch.Size([4, 3])
torch.Size([5, 4])
torch.Size([2, 5])

In [47]:

    

m2 = m.weight.data.T


    

In [55]:

    

torch.ger((m2 > 0).int().view(-1), (m2 > 0).int().view(-1))


    

    
Out[55]:





tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
        [0, 0, 0, 1, 0, 1, 0, 0, 0, 0],
        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
        [0, 0, 0, 1, 0, 1, 0, 0, 0, 0],
        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=torch.int32)

In [52]:

    

m2


    

    
Out[52]:





tensor([[-0.3507, -0.4252],
        [-0.3169,  0.2901],
        [-0.2204,  0.2198],
        [-0.1902, -0.0727],
        [-0.2340, -0.1836]])

In [ ]: