In [20]:

    

%load_ext autoreload
%autoreload 2

import matplotlib.pyplot as plt
from matplotlib import rcParams
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
rcParams['figure.figsize'] = (12,6)

import numpy as np
import pandas as pd
import seaborn as sns

import os

import torch
from torch import nn, optim
from torchvision import models
from nupic.torch.modules import KWinners2d, Flatten
from torchsummary import summary

from utils import Dataset


    

In [27]:

    

#
a = torch.randn(10,10)
print(a.shape)
torch.kthvalue(a.view(-1), 101)


    

    




torch.Size([10, 10])






    




---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-27-dd7453fdfe10> in <module>
      2 a = torch.randn(10,10)
      3 print(a.shape)
----> 4 torch.kthvalue(a.view(-1), 101)

RuntimeError: selected index k out of range

In [16]:

    

dir(np)


    

    
Out[16]:





['ALLOW_THREADS',
 'AxisError',
 'BUFSIZE',
 'CLIP',
 'ComplexWarning',
 'DataSource',
 'ERR_CALL',
 'ERR_DEFAULT',
 'ERR_IGNORE',
 'ERR_LOG',
 'ERR_PRINT',
 'ERR_RAISE',
 'ERR_WARN',
 'FLOATING_POINT_SUPPORT',
 'FPE_DIVIDEBYZERO',
 'FPE_INVALID',
 'FPE_OVERFLOW',
 'FPE_UNDERFLOW',
 'False_',
 'Inf',
 'Infinity',
 'MAXDIMS',
 'MAY_SHARE_BOUNDS',
 'MAY_SHARE_EXACT',
 'MachAr',
 'ModuleDeprecationWarning',
 'NAN',
 'NINF',
 'NZERO',
 'NaN',
 'PINF',
 'PZERO',
 'RAISE',
 'RankWarning',
 'SHIFT_DIVIDEBYZERO',
 'SHIFT_INVALID',
 'SHIFT_OVERFLOW',
 'SHIFT_UNDERFLOW',
 'ScalarType',
 'Tester',
 'TooHardError',
 'True_',
 'UFUNC_BUFSIZE_DEFAULT',
 'UFUNC_PYVALS_NAME',
 'VisibleDeprecationWarning',
 'WRAP',
 '_NoValue',
 '_UFUNC_API',
 '__NUMPY_SETUP__',
 '__all__',
 '__builtins__',
 '__cached__',
 '__config__',
 '__doc__',
 '__file__',
 '__git_revision__',
 '__loader__',
 '__mkl_version__',
 '__name__',
 '__package__',
 '__path__',
 '__spec__',
 '__version__',
 '_add_newdoc_ufunc',
 '_arg',
 '_distributor_init',
 '_globals',
 '_mat',
 '_mklinit',
 '_pytesttester',
 'abs',
 'absolute',
 'absolute_import',
 'add',
 'add_docstring',
 'add_newdoc',
 'add_newdoc_ufunc',
 'alen',
 'all',
 'allclose',
 'alltrue',
 'amax',
 'amin',
 'angle',
 'any',
 'append',
 'apply_along_axis',
 'apply_over_axes',
 'arange',
 'arccos',
 'arccosh',
 'arcsin',
 'arcsinh',
 'arctan',
 'arctan2',
 'arctanh',
 'argmax',
 'argmin',
 'argpartition',
 'argsort',
 'argwhere',
 'around',
 'array',
 'array2string',
 'array_equal',
 'array_equiv',
 'array_repr',
 'array_split',
 'array_str',
 'asanyarray',
 'asarray',
 'asarray_chkfinite',
 'ascontiguousarray',
 'asfarray',
 'asfortranarray',
 'asmatrix',
 'asscalar',
 'atleast_1d',
 'atleast_2d',
 'atleast_3d',
 'average',
 'bartlett',
 'base_repr',
 'binary_repr',
 'bincount',
 'bitwise_and',
 'bitwise_not',
 'bitwise_or',
 'bitwise_xor',
 'blackman',
 'block',
 'bmat',
 'bool',
 'bool8',
 'bool_',
 'broadcast',
 'broadcast_arrays',
 'broadcast_to',
 'busday_count',
 'busday_offset',
 'busdaycalendar',
 'byte',
 'byte_bounds',
 'bytes0',
 'bytes_',
 'c_',
 'can_cast',
 'cast',
 'cbrt',
 'cdouble',
 'ceil',
 'cfloat',
 'char',
 'character',
 'chararray',
 'choose',
 'clip',
 'clongdouble',
 'clongfloat',
 'column_stack',
 'common_type',
 'compare_chararrays',
 'compat',
 'complex',
 'complex128',
 'complex256',
 'complex64',
 'complex_',
 'complexfloating',
 'compress',
 'concatenate',
 'conj',
 'conjugate',
 'convolve',
 'copy',
 'copysign',
 'copyto',
 'core',
 'corrcoef',
 'correlate',
 'cos',
 'cosh',
 'count_nonzero',
 'cov',
 'cross',
 'csingle',
 'ctypeslib',
 'cumprod',
 'cumproduct',
 'cumsum',
 'datetime64',
 'datetime_as_string',
 'datetime_data',
 'deg2rad',
 'degrees',
 'delete',
 'deprecate',
 'deprecate_with_doc',
 'diag',
 'diag_indices',
 'diag_indices_from',
 'diagflat',
 'diagonal',
 'diff',
 'digitize',
 'disp',
 'divide',
 'division',
 'divmod',
 'dot',
 'double',
 'dsplit',
 'dstack',
 'dtype',
 'e',
 'ediff1d',
 'einsum',
 'einsum_path',
 'emath',
 'empty',
 'empty_like',
 'equal',
 'errstate',
 'euler_gamma',
 'exp',
 'exp2',
 'expand_dims',
 'expm1',
 'extract',
 'eye',
 'fabs',
 'fastCopyAndTranspose',
 'fft',
 'fill_diagonal',
 'find_common_type',
 'finfo',
 'fix',
 'flatiter',
 'flatnonzero',
 'flexible',
 'flip',
 'fliplr',
 'flipud',
 'float',
 'float128',
 'float16',
 'float32',
 'float64',
 'float_',
 'float_power',
 'floating',
 'floor',
 'floor_divide',
 'fmax',
 'fmin',
 'fmod',
 'format_float_positional',
 'format_float_scientific',
 'format_parser',
 'frexp',
 'frombuffer',
 'fromfile',
 'fromfunction',
 'fromiter',
 'frompyfunc',
 'fromregex',
 'fromstring',
 'full',
 'full_like',
 'fv',
 'gcd',
 'generic',
 'genfromtxt',
 'geomspace',
 'get_array_wrap',
 'get_include',
 'get_printoptions',
 'getbufsize',
 'geterr',
 'geterrcall',
 'geterrobj',
 'gradient',
 'greater',
 'greater_equal',
 'half',
 'hamming',
 'hanning',
 'heaviside',
 'histogram',
 'histogram2d',
 'histogram_bin_edges',
 'histogramdd',
 'hsplit',
 'hstack',
 'hypot',
 'i0',
 'identity',
 'iinfo',
 'imag',
 'in1d',
 'index_exp',
 'indices',
 'inexact',
 'inf',
 'info',
 'infty',
 'inner',
 'insert',
 'int',
 'int0',
 'int16',
 'int32',
 'int64',
 'int8',
 'int_',
 'int_asbuffer',
 'intc',
 'integer',
 'interp',
 'intersect1d',
 'intp',
 'invert',
 'ipmt',
 'irr',
 'is_busday',
 'isclose',
 'iscomplex',
 'iscomplexobj',
 'isfinite',
 'isfortran',
 'isin',
 'isinf',
 'isnan',
 'isnat',
 'isneginf',
 'isposinf',
 'isreal',
 'isrealobj',
 'isscalar',
 'issctype',
 'issubclass_',
 'issubdtype',
 'issubsctype',
 'iterable',
 'ix_',
 'kaiser',
 'kron',
 'lcm',
 'ldexp',
 'left_shift',
 'less',
 'less_equal',
 'lexsort',
 'lib',
 'linalg',
 'linspace',
 'little_endian',
 'load',
 'loads',
 'loadtxt',
 'log',
 'log10',
 'log1p',
 'log2',
 'logaddexp',
 'logaddexp2',
 'logical_and',
 'logical_not',
 'logical_or',
 'logical_xor',
 'logspace',
 'long',
 'longcomplex',
 'longdouble',
 'longfloat',
 'longlong',
 'lookfor',
 'ma',
 'mafromtxt',
 'mask_indices',
 'mat',
 'math',
 'matmul',
 'matrix',
 'matrixlib',
 'max',
 'maximum',
 'maximum_sctype',
 'may_share_memory',
 'mean',
 'median',
 'memmap',
 'meshgrid',
 'mgrid',
 'min',
 'min_scalar_type',
 'minimum',
 'mintypecode',
 'mirr',
 'mod',
 'modf',
 'moveaxis',
 'msort',
 'multiply',
 'nan',
 'nan_to_num',
 'nanargmax',
 'nanargmin',
 'nancumprod',
 'nancumsum',
 'nanmax',
 'nanmean',
 'nanmedian',
 'nanmin',
 'nanpercentile',
 'nanprod',
 'nanquantile',
 'nanstd',
 'nansum',
 'nanvar',
 'nbytes',
 'ndarray',
 'ndenumerate',
 'ndfromtxt',
 'ndim',
 'ndindex',
 'nditer',
 'negative',
 'nested_iters',
 'newaxis',
 'nextafter',
 'nonzero',
 'not_equal',
 'nper',
 'npv',
 'numarray',
 'number',
 'obj2sctype',
 'object',
 'object0',
 'object_',
 'ogrid',
 'oldnumeric',
 'ones',
 'ones_like',
 'outer',
 'packbits',
 'pad',
 'partition',
 'percentile',
 'pi',
 'piecewise',
 'place',
 'pmt',
 'poly',
 'poly1d',
 'polyadd',
 'polyder',
 'polydiv',
 'polyfit',
 'polyint',
 'polymul',
 'polynomial',
 'polysub',
 'polyval',
 'positive',
 'power',
 'ppmt',
 'print_function',
 'printoptions',
 'prod',
 'product',
 'promote_types',
 'ptp',
 'put',
 'put_along_axis',
 'putmask',
 'pv',
 'quantile',
 'r_',
 'rad2deg',
 'radians',
 'random',
 'rank',
 'rate',
 'ravel',
 'ravel_multi_index',
 'real',
 'real_if_close',
 'rec',
 'recarray',
 'recfromcsv',
 'recfromtxt',
 'reciprocal',
 'record',
 'remainder',
 'repeat',
 'require',
 'reshape',
 'resize',
 'result_type',
 'right_shift',
 'rint',
 'roll',
 'rollaxis',
 'roots',
 'rot90',
 'round',
 'round_',
 'row_stack',
 's_',
 'safe_eval',
 'save',
 'savetxt',
 'savez',
 'savez_compressed',
 'sctype2char',
 'sctypeDict',
 'sctypeNA',
 'sctypes',
 'searchsorted',
 'select',
 'set_numeric_ops',
 'set_printoptions',
 'set_string_function',
 'setbufsize',
 'setdiff1d',
 'seterr',
 'seterrcall',
 'seterrobj',
 'setxor1d',
 'shape',
 'shares_memory',
 'short',
 'show_config',
 'sign',
 'signbit',
 'signedinteger',
 'sin',
 'sinc',
 'single',
 'singlecomplex',
 'sinh',
 'size',
 'sometrue',
 'sort',
 'sort_complex',
 'source',
 'spacing',
 'split',
 'sqrt',
 'square',
 'squeeze',
 'stack',
 'std',
 'str',
 'str0',
 'str_',
 'string_',
 'subtract',
 'sum',
 'swapaxes',
 'sys',
 'take',
 'take_along_axis',
 'tan',
 'tanh',
 'tensordot',
 'test',
 'testing',
 'tile',
 'timedelta64',
 'trace',
 'tracemalloc_domain',
 'transpose',
 'trapz',
 'tri',
 'tril',
 'tril_indices',
 'tril_indices_from',
 'trim_zeros',
 'triu',
 'triu_indices',
 'triu_indices_from',
 'true_divide',
 'trunc',
 'typeDict',
 'typeNA',
 'typecodes',
 'typename',
 'ubyte',
 'ufunc',
 'uint',
 'uint0',
 'uint16',
 'uint32',
 'uint64',
 'uint8',
 'uintc',
 'uintp',
 'ulonglong',
 'unicode',
 'unicode_',
 'union1d',
 'unique',
 'unpackbits',
 'unravel_index',
 'unsignedinteger',
 'unwrap',
 'ushort',
 'vander',
 'var',
 'vdot',
 'vectorize',
 'version',
 'void',
 'void0',
 'vsplit',
 'vstack',
 'warnings',
 'where',
 'who',
 'zeros',
 'zeros_like']

In [18]:

    

not np.isnan(np.mean([]))


    

    
Out[18]:





False

In [152]:

    

# load dataset
dataset = Dataset(dict(dataset_name="CIFAR10", data_dir="~/nta/datasets",
                          stats_mean=(0.4914, 0.4822, 0.4465),
                            stats_std=(0.2023, 0.1994, 0.2010)))


    

In [159]:

    

# load model
from networks import VGG19Small
model = VGG19Small(dict(device='cpu', kwinners=True))
loss_func = nn.CrossEntropyLoss()
# summary(model, (3, 32, 32))


    

In [160]:

    

# get one batch
input, target = next(iter(dataset.train_loader))
input.shape, target.shape


    

    
Out[160]:





(torch.Size([128, 3, 32, 32]), torch.Size([128]))

In [ ]:

    

# run one forward and loss calculation
output = model.forward(input)
loss = loss_func(output, target)
loss


    

    




<class 'torch.nn.modules.conv.Conv2d'>
<class 'nupic.torch.modules.k_winners.KWinners2d'>
<class 'torch.nn.modules.conv.Conv2d'>
<class 'nupic.torch.modules.k_winners.KWinners2d'>
<class 'torch.nn.modules.conv.Conv2d'>

In [ ]:

    

len(model.correlations)


    

In [ ]:

    

for c in model.correlations:
    print(c.shape)


    

In [ ]:

    

32*32*64, 16*16*64, 8*8*64, 4*4*128, 2*2*256, 1*1*512


    

In [141]:

    

# run a full epoch
optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
for inputs, targets in dataset.test_loader:
    optimizer.zero_grad()
    outputs = model(inputs)
    loss = loss_func(outputs, targets)
    loss.backward()
    optimizer.step()


    

    




---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-141-7d8689d6ac90> in <module>
      3 for inputs, targets in dataset.test_loader:
      4     optimizer.zero_grad()
----> 5     outputs = model(inputs)
      6     loss = loss_func(outputs, targets)
      7     loss.backward()

~/miniconda3/envs/numenta/lib/python3.7/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    491             result = self._slow_forward(*input, **kwargs)
    492         else:
--> 493             result = self.forward(*input, **kwargs)
    494         for hook in self._forward_hooks.values():
    495             hook_result = hook(self, input, result)

~/nta/nupic.research/projects/dynamic_sparse/networks.py in forward(self, x)
    264                         self.correlations.append(outer_product)
    265                     else:
--> 266                         self.correlations[idx-1] += outer_product
    267                 prev_act = curr_act
    268             idx += 1

RuntimeError: The size of tensor a (2048) must match the size of tensor b (16384) at non-singleton dimension 1

In [78]:

    

l=kwinners[0]
l.duty_cycle.view(-1)


    

    
Out[78]:





tensor([0.2701, 0.3506, 0.3479, 0.3373, 0.3068, 0.3285, 0.2560, 0.2628, 0.3072,
        0.3014, 0.3104, 0.3046, 0.3292, 0.3081, 0.3436, 0.2817, 0.2963, 0.2982,
        0.2639, 0.2874, 0.3389, 0.2479, 0.2524, 0.2583, 0.2534, 0.3026, 0.3088,
        0.3394, 0.3166, 0.3331, 0.3141, 0.2862, 0.3031, 0.3051, 0.3456, 0.3524,
        0.3028, 0.3090, 0.3048, 0.2549, 0.3194, 0.3124, 0.2668, 0.2460, 0.2610,
        0.3067, 0.3237, 0.3108, 0.3068, 0.2518, 0.3028, 0.3056, 0.3132, 0.3056,
        0.3084, 0.3048, 0.3287, 0.3392, 0.2719, 0.2367, 0.2422, 0.2730, 0.3494,
        0.2920])

In [84]:

    

correlations = []
prev = None
for l in kwinners:
    if prev is None:
        prev = l.duty_cycle.view(-1)
    else:
        nextt = l.duty_cycle.view(-1)
        correlations.append(torch.ger(prev, nextt))
        prev = nextt


    

In [87]:

    

len(correlations)


    

    
Out[87]:





4

In [88]:

    

len(kwinners)


    

    
Out[88]:





5

In [90]:

    

correlations[0].shape


    

    
Out[90]:





torch.Size([64, 64])

In [93]:

    

correlations[-1].shape


    

    
Out[93]:





torch.Size([256, 512])

In [100]:

    

torch.exp(correlations[0][:5,:5])


    

    
Out[100]:





tensor([[1.0908, 1.0807, 1.0834, 1.0854, 1.0831],
        [1.1194, 1.1060, 1.1095, 1.1122, 1.1092],
        [1.1184, 1.1051, 1.1087, 1.1113, 1.1083],
        [1.1146, 1.1018, 1.1052, 1.1078, 1.1048],
        [1.1037, 1.0921, 1.0952, 1.0976, 1.0949]])

In [101]:

    

correlations[0][:5,:5]


    

    
Out[101]:





tensor([[0.0869, 0.0776, 0.0801, 0.0819, 0.0798],
        [0.1128, 0.1007, 0.1039, 0.1064, 0.1036],
        [0.1119, 0.1000, 0.1031, 0.1056, 0.1028],
        [0.1085, 0.0969, 0.1000, 0.1024, 0.0997],
        [0.0987, 0.0881, 0.0909, 0.0931, 0.0907]])

In [4]:

    

import numpy as np
a = np.array([1,2,3,4,5])


    

In [7]:

    

a[[2,3]]


    

    
Out[7]:





array([3, 4])

In [8]:

    

a = [1,2,3,4,5]
a[[2,3]]


    

    




---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-8-9478b8d269fb> in <module>
      1 a = [1,2,3,4,5]
----> 2 a[[2,3]]

TypeError: list indices must be integers or slices, not list

In [ ]: