PPN Dynamic Filtering

In [1]:

    

%matplotlib inline
%load_ext autoreload
%autoreload 2
import matplotlib.pyplot as plt
import numpy as np
np.random.seed(1337)

from keras.datasets import cifar10
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten, Merge, ThresholdedReLU
from keras.layers import Convolution2D, MaxPooling2D, InputLayer
from keras.regularizers import activity_l2
from keras.utils import np_utils
from keras.callbacks import Callback
from keras import backend as K

from ppap.layers import PPDFN, DFN, PPFN
from utils import *


    

    




Using Theano backend.
Using gpu device 0: GeForce GT 630M (CNMeM is disabled, cuDNN not available)

In [2]:

    

# Modified version of keras/examples/mnist_cnn.py

batch_size = 100
nb_classes = 10

# input image dimensions
img_rows, img_cols = 32, 32
# number of convolutional filters to use
nb_filters = 32
# size of pooling area for max pooling
pool_size = (2, 2)
# convolution kernel size
kernel_size = (3, 3)

# the data, shuffled and split between train and test sets
(X_train, y_train), (X_test, y_test) = cifar10.load_data()

# Explicitly set dim ordering to theano
K.set_image_dim_ordering('th')

if K.image_dim_ordering() == 'th':
    X_train = X_train.reshape(X_train.shape[0], 3, img_rows, img_cols)
    X_test = X_test.reshape(X_test.shape[0], 3, img_rows, img_cols)
    input_shape = (3, img_rows, img_cols)
else:
    X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 1)
    X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 1)
    input_shape = (img_rows, img_cols, 3)

X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255
X_test /= 255
print('X_train shape:', X_train.shape)
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')

# convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)


    

    




X_train shape: (50000, 3, 32, 32)
50000 train samples
10000 test samples

In [3]:

    

modelPP = Sequential()
modelPP.add(InputLayer(batch_input_shape=(batch_size, 3, img_rows, img_cols)))
modelPP.add(PPDFN(5))
modelPP.add(Activation('relu'))
modelPP.add(Flatten())
modelPP.add(Dense(nb_classes))
modelPP.add(Activation('softmax'))

model = Sequential()
model.add(InputLayer(batch_input_shape=(batch_size, 3, img_rows, img_cols)))
model.add(DFN(5))
model.add(Activation('relu'))
model.add(Flatten())
model.add(Dense(nb_classes))
model.add(Activation('softmax'))

modelC = Sequential()
modelC.add(InputLayer(batch_input_shape=(batch_size, 3, img_rows, img_cols)))
modelC.add(Convolution2D(3, 5, 5, border_mode="same"))
modelC.add(Activation('relu'))
modelC.add(Flatten())
modelC.add(Dense(nb_classes))
modelC.add(Activation('softmax'))

modelP = Sequential()
modelP.add(InputLayer(batch_input_shape=(batch_size, 3, img_rows, img_cols)))
modelP.add(PPFN(5))
modelP.add(Activation('relu'))
modelP.add(Flatten())
modelP.add(Dense(nb_classes))
modelP.add(Activation('softmax'))


    

In [4]:

    

modelPP.summary()
model.summary()
modelC.summary()
modelP.summary()


    

    




____________________________________________________________________________________________________
Layer (type)                     Output Shape          Param #     Connected to                     
====================================================================================================
input_1 (InputLayer)             (100, 3, 32, 32)      0                                            
____________________________________________________________________________________________________
ppdfn_1 (PPDFN)                  (100, 3, 32, 32)      1170        input_1[0][0]                    
____________________________________________________________________________________________________
activation_1 (Activation)        (100, 3, 32, 32)      0           ppdfn_1[0][0]                    
____________________________________________________________________________________________________
flatten_1 (Flatten)              (100, 3072)           0           activation_1[0][0]               
____________________________________________________________________________________________________
dense_1 (Dense)                  (100, 10)             30730       flatten_1[0][0]                  
____________________________________________________________________________________________________
activation_2 (Activation)        (100, 10)             0           dense_1[0][0]                    
====================================================================================================
Total params: 31900
____________________________________________________________________________________________________
____________________________________________________________________________________________________
Layer (type)                     Output Shape          Param #     Connected to                     
====================================================================================================
input_2 (InputLayer)             (100, 3, 32, 32)      0                                            
____________________________________________________________________________________________________
dfn_1 (DFN)                      (100, 3, 32, 32)      1525        input_2[0][0]                    
____________________________________________________________________________________________________
activation_3 (Activation)        (100, 3, 32, 32)      0           dfn_1[0][0]                      
____________________________________________________________________________________________________
flatten_2 (Flatten)              (100, 3072)           0           activation_3[0][0]               
____________________________________________________________________________________________________
dense_2 (Dense)                  (100, 10)             30730       flatten_2[0][0]                  
____________________________________________________________________________________________________
activation_4 (Activation)        (100, 10)             0           dense_2[0][0]                    
====================================================================================================
Total params: 32255
____________________________________________________________________________________________________
____________________________________________________________________________________________________
Layer (type)                     Output Shape          Param #     Connected to                     
====================================================================================================
input_3 (InputLayer)             (100, 3, 32, 32)      0                                            
____________________________________________________________________________________________________
convolution2d_1 (Convolution2D)  (100, 3, 32, 32)      228         input_3[0][0]                    
____________________________________________________________________________________________________
activation_5 (Activation)        (100, 3, 32, 32)      0           convolution2d_1[0][0]            
____________________________________________________________________________________________________
flatten_3 (Flatten)              (100, 3072)           0           activation_5[0][0]               
____________________________________________________________________________________________________
dense_3 (Dense)                  (100, 10)             30730       flatten_3[0][0]                  
____________________________________________________________________________________________________
activation_6 (Activation)        (100, 10)             0           dense_3[0][0]                    
====================================================================================================
Total params: 30958
____________________________________________________________________________________________________
____________________________________________________________________________________________________
Layer (type)                     Output Shape          Param #     Connected to                     
====================================================================================================
input_4 (InputLayer)             (100, 3, 32, 32)      0                                            
____________________________________________________________________________________________________
ppfn_1 (PPFN)                    (100, 3, 32, 32)      315         input_4[0][0]                    
____________________________________________________________________________________________________
activation_7 (Activation)        (100, 3, 32, 32)      0           ppfn_1[0][0]                     
____________________________________________________________________________________________________
flatten_4 (Flatten)              (100, 3072)           0           activation_7[0][0]               
____________________________________________________________________________________________________
dense_4 (Dense)                  (100, 10)             30730       flatten_4[0][0]                  
____________________________________________________________________________________________________
activation_8 (Activation)        (100, 10)             0           dense_4[0][0]                    
====================================================================================================
Total params: 31045
____________________________________________________________________________________________________

In [5]:

    

test_index = 1
plt.imshow(X_test[test_index].transpose(1, 2, 0))


    

    
Out[5]:





<matplotlib.image.AxesImage at 0x7fa120374b38>






    

In [6]:

    

fs = get_filter(modelPP, X_test[:batch_size])
plt.imshow(filters_image(fs[test_index]))


    

    
Out[6]:





<matplotlib.image.AxesImage at 0x7fa1180db2b0>






    

In [7]:

    

fs = get_filter(model, X_test[:batch_size])
plt.imshow(filters_image(fs[test_index]))


    

    
Out[7]:





<matplotlib.image.AxesImage at 0x7fa10addeb38>






    

In [8]:

    

fs = get_filter(modelP, X_test[:batch_size])
plt.imshow(filters_image(fs[test_index]))


    

    
Out[8]:





<matplotlib.image.AxesImage at 0x7fa120098438>






    

In [9]:

    

modelP.compile(loss='categorical_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])
        
modelP.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=10,
          verbose=1, validation_data=(X_test, Y_test))


    

    




Train on 50000 samples, validate on 10000 samples
Epoch 1/10
50000/50000 [==============================] - 32s - loss: 1.7342 - acc: 0.3882 - val_loss: 1.5757 - val_acc: 0.4482
Epoch 2/10
50000/50000 [==============================] - 31s - loss: 1.4716 - acc: 0.4855 - val_loss: 1.4502 - val_acc: 0.4954
Epoch 3/10
50000/50000 [==============================] - 36s - loss: 1.3879 - acc: 0.5175 - val_loss: 1.4131 - val_acc: 0.5085
Epoch 4/10
50000/50000 [==============================] - 38s - loss: 1.3435 - acc: 0.5324 - val_loss: 1.4020 - val_acc: 0.5115
Epoch 5/10
50000/50000 [==============================] - 35s - loss: 1.3146 - acc: 0.5458 - val_loss: 1.3837 - val_acc: 0.5143
Epoch 6/10
50000/50000 [==============================] - 38s - loss: 1.2791 - acc: 0.5568 - val_loss: 1.3895 - val_acc: 0.5140
Epoch 7/10
50000/50000 [==============================] - 48s - loss: 1.2667 - acc: 0.5607 - val_loss: 1.3584 - val_acc: 0.5329
Epoch 8/10
50000/50000 [==============================] - 42s - loss: 1.2496 - acc: 0.5686 - val_loss: 1.3559 - val_acc: 0.5312
Epoch 9/10
50000/50000 [==============================] - 48s - loss: 1.2316 - acc: 0.5736 - val_loss: 1.3599 - val_acc: 0.5334
Epoch 10/10
50000/50000 [==============================] - 44s - loss: 1.2179 - acc: 0.5786 - val_loss: 1.3407 - val_acc: 0.5382






    
Out[9]:





<keras.callbacks.History at 0x7fa0d2e0aeb8>

In [10]:

    

model.compile(loss='categorical_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])
        
model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=10,
          verbose=1, validation_data=(X_test, Y_test))


    

    




Train on 50000 samples, validate on 10000 samples
Epoch 1/10
50000/50000 [==============================] - 171s - loss: 1.8301 - acc: 0.3468 - val_loss: 1.6803 - val_acc: 0.4054
Epoch 2/10
50000/50000 [==============================] - 156s - loss: 1.6560 - acc: 0.4126 - val_loss: 1.6428 - val_acc: 0.4182
Epoch 3/10
50000/50000 [==============================] - 145s - loss: 1.5931 - acc: 0.4356 - val_loss: 1.6148 - val_acc: 0.4292
Epoch 4/10
50000/50000 [==============================] - 150s - loss: 1.5545 - acc: 0.4503 - val_loss: 1.5938 - val_acc: 0.4326
Epoch 5/10
50000/50000 [==============================] - 183s - loss: 1.5203 - acc: 0.4609 - val_loss: 1.5781 - val_acc: 0.4355
Epoch 6/10
50000/50000 [==============================] - 165s - loss: 1.4992 - acc: 0.4704 - val_loss: 1.5521 - val_acc: 0.4404
Epoch 7/10
50000/50000 [==============================] - 158s - loss: 1.4725 - acc: 0.4814 - val_loss: 1.5569 - val_acc: 0.4500
Epoch 8/10
50000/50000 [==============================] - 162s - loss: 1.4531 - acc: 0.4863 - val_loss: 1.5348 - val_acc: 0.4570
Epoch 9/10
50000/50000 [==============================] - 155s - loss: 1.4364 - acc: 0.4922 - val_loss: 1.5661 - val_acc: 0.4421
Epoch 10/10
50000/50000 [==============================] - 166s - loss: 1.4224 - acc: 0.4979 - val_loss: 1.5423 - val_acc: 0.4620






    
Out[10]:





<keras.callbacks.History at 0x7fa0d0d505c0>

In [11]:

    

modelPP.compile(loss='categorical_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])
        
modelPP.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=10,
          verbose=1, validation_data=(X_test, Y_test))


    

    




Train on 50000 samples, validate on 10000 samples
Epoch 1/10
50000/50000 [==============================] - 152s - loss: 1.6906 - acc: 0.4034 - val_loss: 1.5035 - val_acc: 0.4735
Epoch 2/10
50000/50000 [==============================] - 161s - loss: 1.4046 - acc: 0.5061 - val_loss: 1.3911 - val_acc: 0.5151
Epoch 3/10
50000/50000 [==============================] - 172s - loss: 1.3218 - acc: 0.5381 - val_loss: 1.3819 - val_acc: 0.5199
Epoch 4/10
50000/50000 [==============================] - 174s - loss: 1.2698 - acc: 0.5576 - val_loss: 1.3303 - val_acc: 0.5280
Epoch 5/10
50000/50000 [==============================] - 169s - loss: 1.2323 - acc: 0.5708 - val_loss: 1.3114 - val_acc: 0.5425
Epoch 6/10
50000/50000 [==============================] - 173s - loss: 1.2121 - acc: 0.5777 - val_loss: 1.3099 - val_acc: 0.5401
Epoch 7/10
50000/50000 [==============================] - 161s - loss: 1.1879 - acc: 0.5836 - val_loss: 1.3060 - val_acc: 0.5456
Epoch 8/10
50000/50000 [==============================] - 135s - loss: 1.1735 - acc: 0.5916 - val_loss: 1.2911 - val_acc: 0.5502
Epoch 9/10
50000/50000 [==============================] - 173s - loss: 1.1575 - acc: 0.5961 - val_loss: 1.3253 - val_acc: 0.5379
Epoch 10/10
50000/50000 [==============================] - 154s - loss: 1.1427 - acc: 0.6013 - val_loss: 1.2865 - val_acc: 0.5483






    
Out[11]:





<keras.callbacks.History at 0x7fa0cccfd978>

In [12]:

    

modelC.compile(loss='categorical_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])
        
modelC.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=10,
          verbose=1, validation_data=(X_test, Y_test))


    

    




Train on 50000 samples, validate on 10000 samples
Epoch 1/10
50000/50000 [==============================] - 19s - loss: 1.8269 - acc: 0.3485 - val_loss: 1.6563 - val_acc: 0.4180
Epoch 2/10
50000/50000 [==============================] - 22s - loss: 1.6128 - acc: 0.4330 - val_loss: 1.5847 - val_acc: 0.4392
Epoch 3/10
50000/50000 [==============================] - 20s - loss: 1.5488 - acc: 0.4582 - val_loss: 1.5603 - val_acc: 0.4425
Epoch 4/10
50000/50000 [==============================] - 19s - loss: 1.4975 - acc: 0.4784 - val_loss: 1.5273 - val_acc: 0.4664
Epoch 5/10
50000/50000 [==============================] - 20s - loss: 1.4601 - acc: 0.4910 - val_loss: 1.4789 - val_acc: 0.4839
Epoch 6/10
50000/50000 [==============================] - 22s - loss: 1.4278 - acc: 0.5022 - val_loss: 1.4865 - val_acc: 0.4834
Epoch 7/10
50000/50000 [==============================] - 24s - loss: 1.4015 - acc: 0.5094 - val_loss: 1.4520 - val_acc: 0.4885
Epoch 8/10
50000/50000 [==============================] - 26s - loss: 1.3877 - acc: 0.5153 - val_loss: 1.4450 - val_acc: 0.4904
Epoch 9/10
50000/50000 [==============================] - 26s - loss: 1.3700 - acc: 0.5216 - val_loss: 1.4379 - val_acc: 0.4935
Epoch 10/10
50000/50000 [==============================] - 23s - loss: 1.3603 - acc: 0.5254 - val_loss: 1.4331 - val_acc: 0.4964






    
Out[12]:





<keras.callbacks.History at 0x7fa0cbe74128>

In [13]:

    

fs = get_filter(model, X_test[:batch_size])
plt.imshow(filters_image(fs[test_index]))


    

    
Out[13]:





<matplotlib.image.AxesImage at 0x7fa0cb9df8d0>






    

In [14]:

    

fs = get_filter(modelPP, X_test[:batch_size])
plt.imshow(filters_image(fs[test_index]))


    

    
Out[14]:





<matplotlib.image.AxesImage at 0x7fa0cb80aef0>






    

In [15]:

    

fs = get_filter(modelP, X_test[:batch_size])
plt.imshow(filters_image(fs[test_index]))


    

    
Out[15]:





<matplotlib.image.AxesImage at 0x7fa0cb775ac8>






    

In [ ]: