In [53]:

    

%matplotlib inline
from matplotlib import pyplot as plt
from keras.models import Sequential
from keras.layers.convolutional import Convolution2D, MaxPooling2D
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.optimizers import SGD
from keras.utils import np_utils
from sklearn.cross_validation import train_test_split
from sklearn.externals import joblib
from sklearn.svm import SVC
from sklearn.metrics import accuracy_score
from sklearn.decomposition import RandomizedPCA
import numpy as np
import pandas as pd
from time import time


    

In [15]:

    

np.set_printoptions(precision=4)
np.set_printoptions(suppress=True)


    

In [16]:

    

nb_classes = 9
batch_size = 64
nb_epoch = 8
np.random.seed(1337) # for reproducibility


    

In [30]:

    

features = joblib.load("./mldata/features_1200.mat")
labels = joblib.load("./mldata/lables_1200.mat")

features = np.array(features, 'int16')
labels = np.array(labels, 'int')

def scale(X, eps = 0.001):
    # scale the data points s.t the columns of the feature space
    # (i.e the predictors) are within the range [0, 1]
    return (X - np.min(X, axis = 0)) / (np.max(X, axis = 0) + eps)

features = features.astype("float32")
features = scale(features)

# scale the data to the range [0, 1] and then construct the training
# and testing splits
(trainX, testX, trainY, testY) = train_test_split(features, labels, test_size = 0.4)
(valX, testX, valY, testY) = train_test_split(testX, testY, test_size = 0.5)

# reshape for convolutions
trainX = trainX.reshape((trainX.shape[0], 1, 28, 28))
testX = testX.reshape((testX.shape[0], 1, 28, 28))
valX = valX.reshape((valX.shape[0], 1, 28, 28))
# convert class vectors to binary class matrices
trainY = np_utils.to_categorical(trainY, nb_classes)
testY = np_utils.to_categorical(testY, nb_classes)
valY = np_utils.to_categorical(valY, nb_classes)


    

In [18]:

    

# Creating the model which consists of 3 conv layers followed by
# 2 fully conntected layers

# Sequential wrapper model
model = Sequential()

# first convolutional layer
model.add(Convolution2D(32, 1, 2, 2))
model.add(Activation('relu'))

# second convolutional layer
model.add(Convolution2D(48, 32, 2, 2))
model.add(Activation('relu')) 
model.add(MaxPooling2D(poolsize=(2,2)))

# third convolutional layer
model.add(Convolution2D(32, 48, 2, 2))
model.add(Activation('relu'))
model.add(MaxPooling2D(poolsize=(2,2)))

# convert convolutional filters to flatt so they can be feed to 
# fully connected layers
model.add(Flatten())

# first fully connected layer
model.add(Dense(32*6*6, 144, init='lecun_uniform'))
model.add(Activation('relu'))
model.add(Dropout(0.5))

# second fully connected layer
model.add(Dense(144, 144, init='lecun_uniform'))
model.add(Activation('relu'))
model.add(Dropout(0.5))

# last fully connected layer which output classes
model.add(Dense(144, 9, init='lecun_uniform'))
model.add(Activation('softmax'))

# setting sgd optimizer parameters
sgd = SGD(lr=0.05, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy', optimizer=sgd)

model.fit(trainX, trainY,
          batch_size=batch_size,
          nb_epoch=nb_epoch,
          show_accuracy=True,
          verbose=1,
          validation_data=(testX, testY))


    

    




Train on 3600 samples, validate on 1200 samples
Epoch 0
3600/3600 [==============================] - 150s - loss: 2.1789 - acc.: 0.1486 - val. loss: 2.1543 - val. acc.: 0.1606
Epoch 1
3600/3600 [==============================] - 141s - loss: 2.0878 - acc.: 0.2564 - val. loss: 1.4304 - val. acc.: 0.4959
Epoch 2
3600/3600 [==============================] - 143s - loss: 0.7865 - acc.: 0.7636 - val. loss: 0.3048 - val. acc.: 0.9158
Epoch 3
3600/3600 [==============================] - 138s - loss: 0.3807 - acc.: 0.9044 - val. loss: 0.3089 - val. acc.: 0.9232
Epoch 4
3600/3600 [==============================] - 141s - loss: 0.3342 - acc.: 0.9186 - val. loss: 0.2908 - val. acc.: 0.9274
Epoch 5
3600/3600 [==============================] - 138s - loss: 0.3183 - acc.: 0.9211 - val. loss: 0.2675 - val. acc.: 0.9317
Epoch 6
3600/3600 [==============================] - 138s - loss: 0.2851 - acc.: 0.9353 - val. loss: 0.2671 - val. acc.: 0.9309
Epoch 7
3600/3600 [==============================] - 444s - loss: 0.2720 - acc.: 0.9342 - val. loss: 0.2760 - val. acc.: 0.9317






    
Out[18]:





{'acc': [0.14861111111111111,
  0.25638888888888889,
  0.76361111111111113,
  0.9044444444444445,
  0.91861111111111116,
  0.9211111111111111,
  0.93527777777777776,
  0.9341666666666667],
 'epoch': [0, 1, 2, 3, 4, 5, 6, 7],
 'loss': [2.1789463927896247,
  2.0878450813488958,
  0.78650011038076728,
  0.3807269037923145,
  0.33419231585977621,
  0.31834055378414006,
  0.28511646862568929,
  0.27201855536819686],
 'val_acc': [0.16063596491228072,
  0.49588815789473684,
  0.9158442982456141,
  0.9232456140350878,
  0.9273574561403509,
  0.9317434210526315,
  0.930921052631579,
  0.9317434210526315],
 'val_loss': [2.154319376109021,
  1.4303748750302472,
  0.30476048631872327,
  0.308910406799675,
  0.2908024769439173,
  0.26752330177746464,
  0.2671278250036357,
  0.2760488027704364]}

In [19]:

    

model.layers


    

    
Out[19]:





[<keras.layers.convolutional.Convolution2D at 0x122e97fd0>,
 <keras.layers.core.Activation at 0x118dd14d0>,
 <keras.layers.convolutional.Convolution2D at 0x118dd1d50>,
 <keras.layers.core.Activation at 0x11ca52590>,
 <keras.layers.convolutional.MaxPooling2D at 0x11ca67390>,
 <keras.layers.convolutional.Convolution2D at 0x11ca52c90>,
 <keras.layers.core.Activation at 0x11ca678d0>,
 <keras.layers.convolutional.MaxPooling2D at 0x11ca71f10>,
 <keras.layers.core.Flatten at 0x11ca498d0>,
 <keras.layers.core.Dense at 0x11ca86ed0>,
 <keras.layers.core.Activation at 0x11ca71b90>,
 <keras.layers.core.Dropout at 0x11ca90950>,
 <keras.layers.core.Dense at 0x11caa5190>,
 <keras.layers.core.Activation at 0x11caa5050>,
 <keras.layers.core.Dropout at 0x11cab97d0>,
 <keras.layers.core.Dense at 0x11cab9c50>,
 <keras.layers.core.Activation at 0x11caa5e90>]

In [20]:

    

for l in range(len(model.layers)):
    if model.layers[l].get_weights() != []:
        print l
        print model.layers[l]
#         print model.layers[l].get_weights()


    

    




0
<keras.layers.convolutional.Convolution2D object at 0x122e97fd0>
2
<keras.layers.convolutional.Convolution2D object at 0x118dd1d50>
5
<keras.layers.convolutional.Convolution2D object at 0x11ca52c90>
9
<keras.layers.core.Dense object at 0x11ca86ed0>
12
<keras.layers.core.Dense object at 0x11caa5190>
15
<keras.layers.core.Dense object at 0x11cab9c50>

In [21]:

    

# Sequential wrapper model
model1 = Sequential()

# first convolutional layer
model1.add(Convolution2D(32, 1, 2, 2, weights=model.layers[0].get_weights()))
model1.add(Activation('relu'))

# second convolutional layer
model1.add(Convolution2D(48, 32, 2, 2, weights=model.layers[2].get_weights()))
model1.add(Activation('relu')) 
model1.add(MaxPooling2D(poolsize=(2,2)))

# third convolutional layer
model1.add(Convolution2D(32, 48, 2, 2, weights=model.layers[5].get_weights()))
model1.add(Activation('relu'))
model1.add(MaxPooling2D(poolsize=(2,2)))

# convert convolutional filters to flatt so they can be feed to 
# fully connected layers
model1.add(Flatten())

# first fully connected layer
model1.add(Dense(32*6*6, 144, init='lecun_uniform', weights=model.layers[9].get_weights()))
model1.add(Activation('relu'))
model1.add(Dropout(0.5))

# second fully connected layer
model1.add(Dense(144, 144, init='lecun_uniform', weights=model.layers[12].get_weights()))
model1.add(Activation('relu'))
model1.add(Dropout(0.5))

model1.compile(loss='categorical_crossentropy', optimizer=sgd)


    

In [26]:

    

features = joblib.load("./mldata/features_1200.mat")
labels = joblib.load("./mldata/lables_1200.mat")

features = np.array(features, 'int16')
labels = np.array(labels, 'int')

features = features.astype("float32")
features = scale(features)

(trainX, testX, trainY, testY) = train_test_split(features, labels, test_size = 0.3)


    

In [47]:

    

trainX_deep = model1.predict(trainX)
testX_deep = model1.predict(testX)


    

    




3600/3600 [==============================] - 39s    
1200/1200 [==============================] - 13s    

In [49]:

    

# dimension reduction by CNN : 144 
t0 = time()
clf = SVC(cache_size=1000, kernel="rbf", C=10.0, gamma=0.03125)

labels_train = np_utils.categorical_probas_to_classes(trainY)
labels_test = np_utils.categorical_probas_to_classes(testY)

clf.fit(trainX_deep, labels_train)

y_pred = clf.predict(testX_deep)
score_accuracy = accuracy_score(y_pred, labels_test, normalize=True)
print "escape time : ", round(time()-t0, 3), "s"
print "accuracy is %s" % score_accuracy


    

    




escape time :  0.615 s
accuracy is 0.9375

In [51]:

    

# dimension : 784 
(trainX, testX, trainY, testY) = train_test_split(features, labels, test_size = 0.4)
(valX, testX, valY, testY) = train_test_split(testX, testY, test_size = 0.5)

t0 = time()
clf = SVC(cache_size=1000, kernel="rbf", C=10.0, gamma=0.03125)

clf.fit(trainX, trainY)

y_pred = clf.predict(testX)
score_accuracy = accuracy_score(y_pred, testY, normalize=True)
print "escape time : ", round(time()-t0, 3), "s"
print "accuracy is %s" % score_accuracy


    

    




escape time :  8.029 s
accuracy is 0.9275

In [55]:

    

# dimension reduction by RandomizedPCA : 144 

pca = RandomizedPCA(n_components=144)

trainX = pca.fit_transform(trainX, trainY)
testX = pca.fit_transform(testX, testY)

t0 = time()
clf = SVC(cache_size=1000, kernel="rbf", C=10.0, gamma=0.03125)

clf.fit(trainX, trainY)

y_pred = clf.predict(testX)
score_accuracy = accuracy_score(y_pred, testY, normalize=True)
print "escape time : ", round(time()-t0, 3), "s"
print "accuracy is %s" % score_accuracy


    

    




escape time :  1.468 s
accuracy is 0.574166666667