In [3]:

    

%matplotlib inline
from matplotlib import pyplot as plt
from keras.models import Sequential
from keras.layers.convolutional import Convolution2D, MaxPooling2D
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.optimizers import SGD
from keras.utils import np_utils
from sklearn.cross_validation import train_test_split
from sklearn.externals import joblib
import numpy as np
import pandas as pd


    

In [4]:

    

np.set_printoptions(precision=4)
np.set_printoptions(suppress=True)


    

In [5]:

    

nb_classes = 9
batch_size = 64
nb_epoch = 1
np.random.seed(1337) # for reproducibility


    

In [6]:

    

features = joblib.load("./mldata/features_1200.mat")
labels = joblib.load("./mldata/lables_1200.mat")

features = np.array(features, 'int16')
labels = np.array(labels, 'int')

def scale(X, eps = 0.001):
    # scale the data points s.t the columns of the feature space
    # (i.e the predictors) are within the range [0, 1]
    return (X - np.min(X, axis = 0)) / (np.max(X, axis = 0) + eps)

features = features.astype("float32")
features = scale(features)

# scale the data to the range [0, 1] and then construct the training
# and testing splits
(trainX, testX, trainY, testY) = train_test_split(features, labels, test_size = 0.4)
(valX, testX, valY, testY) = train_test_split(testX, testY, test_size = 0.5)

# reshape for convolutions
trainX = trainX.reshape((trainX.shape[0], 1, 28, 28))
testX = testX.reshape((testX.shape[0], 1, 28, 28))
valX = valX.reshape((valX.shape[0], 1, 28, 28))

# convert class vectors to binary class matrices
trainY = np_utils.to_categorical(trainY, nb_classes)
testY = np_utils.to_categorical(testY, nb_classes)
valY = np_utils.to_categorical(valY, nb_classes)


    

In [7]:

    

# Creating the model which consists of 3 conv layers followed by
# 2 fully conntected layers

# Sequential wrapper model
model = Sequential()

# first convolutional layer
model.add(Convolution2D(32, 1, 2, 2))
model.add(Activation('relu'))

# second convolutional layer
model.add(Convolution2D(48, 32, 2, 2))
model.add(Activation('relu')) 
model.add(MaxPooling2D(poolsize=(2,2)))

# third convolutional layer
model.add(Convolution2D(32, 48, 2, 2))
model.add(Activation('relu'))
model.add(MaxPooling2D(poolsize=(2,2)))

# convert convolutional filters to flatt so they can be feed to 
# fully connected layers
model.add(Flatten())

# first fully connected layer
model.add(Dense(32*6*6, 144, init='lecun_uniform'))
model.add(Activation('relu'))
model.add(Dropout(0.5))

# second fully connected layer
model.add(Dense(144, 144, init='lecun_uniform'))
model.add(Activation('relu'))
model.add(Dropout(0.5))

# last fully connected layer which output classes
model.add(Dense(144, 9, init='lecun_uniform'))
model.add(Activation('softmax'))

# setting sgd optimizer parameters
sgd = SGD(lr=0.05, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy', optimizer=sgd)

model.fit(trainX, trainY,
          batch_size=batch_size,
          nb_epoch=nb_epoch,
          show_accuracy=True,
          verbose=1,
          validation_data=(testX, testY))


    

    




Train on 3600 samples, validate on 1200 samples
Epoch 0
3600/3600 [==============================] - 135s - loss: 2.1788 - acc.: 0.1497 - val. loss: 2.1512 - val. acc.: 0.1606






    
Out[7]:





{'acc': [0.14972222222222223],
 'epoch': [0],
 'loss': [2.1788399429410532],
 'val_acc': [0.16063596491228072],
 'val_loss': [2.1511703137799727]}

In [8]:

    

model.layers


    

    
Out[8]:





[<keras.layers.convolutional.Convolution2D at 0x10a0c4850>,
 <keras.layers.core.Activation at 0x104505050>,
 <keras.layers.convolutional.Convolution2D at 0x104505850>,
 <keras.layers.core.Activation at 0x10f1c9510>,
 <keras.layers.convolutional.MaxPooling2D at 0x10f1c9810>,
 <keras.layers.convolutional.Convolution2D at 0x10f1c9610>,
 <keras.layers.core.Activation at 0x10f1c9c10>,
 <keras.layers.convolutional.MaxPooling2D at 0x10f1c9e90>,
 <keras.layers.core.Flatten at 0x10f1c9c90>,
 <keras.layers.core.Dense at 0x10f1c9fd0>,
 <keras.layers.core.Activation at 0x10f1c9ed0>,
 <keras.layers.core.Dropout at 0x10f1d4250>,
 <keras.layers.core.Dense at 0x10f1d42d0>,
 <keras.layers.core.Activation at 0x10f1d4090>,
 <keras.layers.core.Dropout at 0x10f1d4550>,
 <keras.layers.core.Dense at 0x10f1d45d0>,
 <keras.layers.core.Activation at 0x10f1d4350>]

In [9]:

    

for l in range(len(model.layers)):
    if model.layers[l].get_weights() != []:
        print l
        print model.layers[l]
#         print model.layers[l].get_weights()


    

    




0
<keras.layers.convolutional.Convolution2D object at 0x10a0c4850>
2
<keras.layers.convolutional.Convolution2D object at 0x104505850>
5
<keras.layers.convolutional.Convolution2D object at 0x10f1c9610>
9
<keras.layers.core.Dense object at 0x10f1c9fd0>
12
<keras.layers.core.Dense object at 0x10f1d42d0>
15
<keras.layers.core.Dense object at 0x10f1d45d0>

In [10]:

    

model.layers[0].get_weights()[0].shape


    

    
Out[10]:





(32, 1, 2, 2)

In [11]:

    

valX.shape


    

    
Out[11]:





(1200, 1, 28, 28)

In [12]:

    

valX[0].ravel().reshape(28,28).shape


    

    
Out[12]:





(28, 28)

In [13]:

    

sample_0 = valX[0].ravel().reshape(28,28)
sample_1 = valX[1].ravel().reshape(28,28)
sample_2 = valX[2].ravel().reshape(28,28)
sample_3 = valX[3].ravel().reshape(28,28)
sample_4 = valX[4].ravel().reshape(28,28)

fig, (ax0, ax1, ax2, ax3, ax4) = plt.subplots(1, 5)
ax0.imshow(sample_0, cmap='gray', interpolation='nearest')
ax1.imshow(sample_1, cmap='gray')
ax2.imshow(sample_2, cmap='gray')
ax3.imshow(sample_3, cmap='gray')
ax4.imshow(sample_4, cmap='gray')


    

    
Out[13]:





<matplotlib.image.AxesImage at 0x10a284950>






    

In [14]:

    

model1 = Sequential()
model1.add(Convolution2D(32, 1, 2, 2, weights=model.layers[0].get_weights()))
model1.add(Activation('relu'))
model1.compile(loss='categorical_crossentropy', optimizer=sgd)


    

In [15]:

    

valX_h_1 = model1.predict(valX)


    

    




1200/1200 [==============================] - 0s     

In [16]:

    

valX_h_1.shape


    

    
Out[16]:





(1200, 32, 27, 27)

In [17]:

    

valX_h_1[0][0].shape


    

    
Out[17]:





(27, 27)

In [18]:

    

sample_0 = valX_h_1[0][0]
sample_1 = valX_h_1[0][1]
sample_2 = valX_h_1[0][2]
sample_3 = valX_h_1[0][3]
sample_4 = valX_h_1[0][4]
sample_5 = valX_h_1[0][5]
sample_6 = valX_h_1[0][6]
sample_7 = valX_h_1[0][7]
sample_8 = valX_h_1[0][8]
sample_9 = valX_h_1[0][9]
sample_10 = valX_h_1[0][10]
sample_11 = valX_h_1[0][11]
sample_12 = valX_h_1[0][12]
sample_13 = valX_h_1[0][13]
sample_14 = valX_h_1[0][14]
sample_15 = valX_h_1[0][15]
sample_16 = valX_h_1[0][16]
sample_17 = valX_h_1[0][17]
sample_18 = valX_h_1[0][18]
sample_19 = valX_h_1[0][19]
sample_20 = valX_h_1[0][20]
sample_21 = valX_h_1[0][21]
sample_22 = valX_h_1[0][22]
sample_23 = valX_h_1[0][23]
sample_24 = valX_h_1[0][24]
sample_25 = valX_h_1[0][25]
sample_26 = valX_h_1[0][26]
sample_27 = valX_h_1[0][27]
sample_28 = valX_h_1[0][28]
sample_29 = valX_h_1[0][29]
sample_30 = valX_h_1[0][30]
sample_31 = valX_h_1[0][31]

fig, (ax0, ax1, ax2, ax3, ax4) = plt.subplots(1, 5)
ax0.imshow(sample_0, cmap='gray')
ax1.imshow(sample_1, cmap='gray')
ax2.imshow(sample_2, cmap='gray')
ax3.imshow(sample_3, cmap='gray')
ax4.imshow(sample_4, cmap='gray')

fig, (ax0, ax1, ax2, ax3, ax4) = plt.subplots(1, 5)
ax0.imshow(sample_10, cmap='gray')
ax1.imshow(sample_11, cmap='gray')
ax2.imshow(sample_12, cmap='gray')
ax3.imshow(sample_13, cmap='gray')
ax4.imshow(sample_14, cmap='gray')

fig, (ax0, ax1, ax2, ax3, ax4) = plt.subplots(1, 5)
ax0.imshow(sample_20, cmap='gray')
ax1.imshow(sample_21, cmap='gray')
ax2.imshow(sample_22, cmap='gray')
ax3.imshow(sample_23, cmap='gray')
ax4.imshow(sample_24, cmap='gray')


    

    
Out[18]:





<matplotlib.image.AxesImage at 0x119b72e90>






    













    













    

In [19]:

    

# Sequential wrapper model
model2 = Sequential()

# first convolutional layer
model2.add(Convolution2D(32, 1, 2, 2, weights=model.layers[0].get_weights()))
model2.add(Activation('relu'))

# second convolutional layer
model2.add(Convolution2D(48, 32, 2, 2, weights=model.layers[2].get_weights()))
model2.add(Activation('relu')) 
model2.add(MaxPooling2D(poolsize=(2,2)))

model2.compile(loss='categorical_crossentropy', optimizer=sgd)


    

In [20]:

    

valX_h_2 = model2.predict(valX)


    

    




1200/1200 [==============================] - 9s     

In [21]:

    

valX_h_2.shape


    

    
Out[21]:





(1200, 48, 13, 13)

In [22]:

    

valX_h_2[0].shape


    

    
Out[22]:





(48, 13, 13)

In [23]:

    

valX_h_2[0][0].shape


    

    
Out[23]:





(13, 13)

In [24]:

    

sample_0 = valX_h_2[0][0]
sample_1 = valX_h_2[0][1]
sample_2 = valX_h_2[0][2]
sample_3 = valX_h_2[0][3]
sample_4 = valX_h_2[0][4]
sample_5 = valX_h_2[0][5]
sample_6 = valX_h_2[0][6]
sample_7 = valX_h_2[0][7]
sample_8 = valX_h_2[0][8]
sample_9 = valX_h_2[0][9]
sample_10 = valX_h_2[0][10]
sample_11 = valX_h_2[0][11]
sample_12 = valX_h_2[0][12]
sample_13 = valX_h_2[0][13]
sample_14 = valX_h_2[0][14]
sample_15 = valX_h_2[0][15]
sample_16 = valX_h_2[0][16]
sample_17 = valX_h_2[0][17]
sample_18 = valX_h_2[0][18]
sample_19 = valX_h_2[0][19]
sample_20 = valX_h_2[0][20]
sample_21 = valX_h_2[0][21]
sample_22 = valX_h_2[0][22]
sample_23 = valX_h_2[0][23]
sample_24 = valX_h_2[0][24]
sample_25 = valX_h_2[0][25]
sample_26 = valX_h_2[0][26]
sample_27 = valX_h_2[0][27]
sample_28 = valX_h_2[0][28]
sample_29 = valX_h_2[0][29]
sample_30 = valX_h_2[0][30]
sample_31 = valX_h_2[0][31]

fig, (ax0, ax1, ax2, ax3, ax4) = plt.subplots(1, 5)
ax0.imshow(sample_0, cmap='gray')
ax1.imshow(sample_1, cmap='gray')
ax2.imshow(sample_2, cmap='gray')
ax3.imshow(sample_3, cmap='gray')
ax4.imshow(sample_4, cmap='gray')

fig, (ax0, ax1, ax2, ax3, ax4) = plt.subplots(1, 5)
ax0.imshow(sample_10, cmap='gray')
ax1.imshow(sample_11, cmap='gray')
ax2.imshow(sample_12, cmap='gray')
ax3.imshow(sample_13, cmap='gray')
ax4.imshow(sample_14, cmap='gray')

fig, (ax0, ax1, ax2, ax3, ax4) = plt.subplots(1, 5)
ax0.imshow(sample_20, cmap='gray')
ax1.imshow(sample_21, cmap='gray')
ax2.imshow(sample_22, cmap='gray')
ax3.imshow(sample_23, cmap='gray')
ax4.imshow(sample_24, cmap='gray')


    

    
Out[24]:





<matplotlib.image.AxesImage at 0x131d16f10>






    













    













    

In [25]:

    

# Sequential wrapper model
model3 = Sequential()

# first convolutional layer
model3.add(Convolution2D(32, 1, 2, 2, weights=model.layers[0].get_weights()))
model3.add(Activation('relu'))

# second convolutional layer
model3.add(Convolution2D(48, 32, 2, 2, weights=model.layers[2].get_weights()))
model3.add(Activation('relu')) 
model3.add(MaxPooling2D(poolsize=(2,2)))

# third convolutional layer
model3.add(Convolution2D(32, 48, 2, 2, weights=model.layers[5].get_weights()))
model3.add(Activation('relu'))
model3.add(MaxPooling2D(poolsize=(2,2)))
model3.compile(loss='categorical_crossentropy', optimizer=sgd)


    

In [26]:

    

valX_h_3 = model3.predict(valX)


    

    




1200/1200 [==============================] - 11s    

In [27]:

    

valX_h_3.shape


    

    
Out[27]:





(1200, 32, 6, 6)

In [28]:

    

valX_h_3[0][0].shape


    

    
Out[28]:





(6, 6)

In [29]:

    

sample_0 = valX_h_3[0][0]
sample_1 = valX_h_3[0][1]
sample_2 = valX_h_3[0][2]
sample_3 = valX_h_2[0][3]
sample_4 = valX_h_3[0][4]
sample_5 = valX_h_3[0][5]
sample_6 = valX_h_3[0][6]
sample_7 = valX_h_3[0][7]
sample_8 = valX_h_3[0][8]
sample_9 = valX_h_3[0][9]
sample_10 = valX_h_3[0][10]
sample_11 = valX_h_3[0][11]
sample_12 = valX_h_3[0][12]
sample_13 = valX_h_3[0][13]
sample_14 = valX_h_3[0][14]
sample_15 = valX_h_3[0][15]
sample_16 = valX_h_3[0][16]
sample_17 = valX_h_3[0][17]
sample_18 = valX_h_3[0][18]
sample_19 = valX_h_3[0][19]
sample_20 = valX_h_3[0][20]
sample_21 = valX_h_3[0][21]
sample_22 = valX_h_3[0][22]
sample_23 = valX_h_3[0][23]
sample_24 = valX_h_3[0][24]
sample_25 = valX_h_3[0][25]
sample_26 = valX_h_3[0][26]
sample_27 = valX_h_3[0][27]
sample_28 = valX_h_3[0][28]
sample_29 = valX_h_3[0][29]
sample_30 = valX_h_3[0][30]
sample_31 = valX_h_3[0][31]

fig, (ax0, ax1, ax2, ax3, ax4) = plt.subplots(1, 5)
ax0.imshow(sample_0, cmap='gray')
ax1.imshow(sample_1, cmap='gray')
ax2.imshow(sample_2, cmap='gray')
ax3.imshow(sample_3, cmap='gray')
ax4.imshow(sample_4, cmap='gray')

fig, (ax0, ax1, ax2, ax3, ax4) = plt.subplots(1, 5)
ax0.imshow(sample_10, cmap='gray')
ax1.imshow(sample_11, cmap='gray')
ax2.imshow(sample_12, cmap='gray')
ax3.imshow(sample_13, cmap='gray')
ax4.imshow(sample_14, cmap='gray')

fig, (ax0, ax1, ax2, ax3, ax4) = plt.subplots(1, 5)
ax0.imshow(sample_20, cmap='gray')
ax1.imshow(sample_21, cmap='gray')
ax2.imshow(sample_22, cmap='gray')
ax3.imshow(sample_23, cmap='gray')
ax4.imshow(sample_24, cmap='gray')


    

    
Out[29]:





<matplotlib.image.AxesImage at 0x131d7c910>






    













    













    

In [30]:

    

# Creating the model which consists of 3 conv layers followed by
# 2 fully conntected layers

# Sequential wrapper model
model4 = Sequential()

# first convolutional layer
model4.add(Convolution2D(32, 1, 2, 2, weights=model.layers[0].get_weights()))
model4.add(Activation('relu'))

# second convolutional layer
model4.add(Convolution2D(48, 32, 2, 2, weights=model.layers[2].get_weights()))
model4.add(Activation('relu')) 
model4.add(MaxPooling2D(poolsize=(2,2)))

# third convolutional layer
model4.add(Convolution2D(32, 48, 2, 2, weights=model.layers[5].get_weights()))
model4.add(Activation('relu'))
model4.add(MaxPooling2D(poolsize=(2,2)))

# convert convolutional filters to flatt so they can be feed to 
# fully connected layers
model4.add(Flatten())

model4.compile(loss='categorical_crossentropy', optimizer=sgd)


    

In [31]:

    

valX_c_0 = model4.predict(valX)


    

    




1200/1200 [==============================] - 11s    

In [32]:

    

32*6*6


    

    
Out[32]:





1152

In [33]:

    

valX_c_0.shape


    

    
Out[33]:





(1200, 1152)

In [40]:

    

valX_c_0[0]


    

    
Out[40]:





array([-0.    , -0.    , -0.    , ...,  0.5077,  0.3446,  0.3877])

In [41]:

    

# df = pd.DataFrame(valX_c_0[0])
df = pd.DataFrame({'fully connected layers' : valX_c_0[0]})


    

In [42]:

    

df.head(5)


    

    
Out[42]:






  

    

      

      
fully connected layers
    
  
  

    

      
0
      
-5.529431e-18
    
    

      
1
      
-5.529431e-18
    
    

      
2
      
-5.529431e-18
    
    

      
3
      
-5.529431e-18
    
    

      
4
      
-5.529431e-18
    
  

In [43]:

    

df.plot(color='DarkGreen')


    

    
Out[43]:





<matplotlib.axes._subplots.AxesSubplot at 0x13711a290>