notebook.community

Edit and run



In [1]:

    
%matplotlib inline
from matplotlib import pyplot as plt
from keras.models import Sequential
from keras.layers.convolutional import Convolution2D, MaxPooling2D
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.optimizers import SGD
from keras.utils import np_utils
from sklearn.cross_validation import train_test_split
from sklearn.externals import joblib
from sklearn.decomposition import RandomizedPCA
import numpy as np
import pandas as pd



In [2]:

    
np.set_printoptions(precision=4)
np.set_printoptions(suppress=True)



In [3]:

    
nb_classes = 9
batch_size = 64
nb_epoch = 4
np.random.seed(1337) # for reproducibility



In [4]:

    
features = joblib.load("./mldata/features_1200.mat")
labels = joblib.load("./mldata/lables_1200.mat")

features = np.array(features, 'int16')
labels = np.array(labels, 'int')

def scale(X, eps = 0.001):
    # scale the data points s.t the columns of the feature space
    # (i.e the predictors) are within the range [0, 1]
    return (X - np.min(X, axis = 0)) / (np.max(X, axis = 0) + eps)

features = features.astype("float32")
features = scale(features)

# scale the data to the range [0, 1] and then construct the training
# and testing splits
(trainX, testX, trainY, testY) = train_test_split(features, labels, test_size = 0.4)
(valX, testX, valY, testY) = train_test_split(testX, testY, test_size = 0.5)



In [5]:

    
print valX.shape
print valY.shape









    



(1200, 784)
(1200,)



In [6]:

    
pca = RandomizedPCA(n_components=2)
valX_pca = pca.fit_transform(valX)



In [9]:

    
print valX_pca[0]
print valX_pca.shape









    



[-2.8117 -4.3638]
(1200, 2)



In [10]:

    
df1 = pd.DataFrame(valX_pca)
df1['digits'] = valY
df1.columns = ['x', 'y', 'digits']



In [11]:

    
df1.head(5)



In [32]:

    
# color example
# http://matplotlib.org/examples/color/named_colors.html
ax = df1[df1['digits'] == 0].plot(kind='scatter', x='x', y='y', color='blue', label='0')
ax = df1[df1['digits'] == 1].plot(kind='scatter', x='x', y='y', color='green', label='1', ax=ax)
ax = df1[df1['digits'] == 2].plot(kind='scatter', x='x', y='y', color='red', label='2', ax=ax)
ax = df1[df1['digits'] == 3].plot(kind='scatter', x='x', y='y', color='cyan', label='3', ax=ax)
ax = df1[df1['digits'] == 4].plot(kind='scatter', x='x', y='y', color='magenta', label='4', ax=ax)
ax = df1[df1['digits'] == 5].plot(kind='scatter', x='x', y='y', color='yellow', label='5', ax=ax)
ax = df1[df1['digits'] == 6].plot(kind='scatter', x='x', y='y', color='black', label='6', ax=ax)
ax = df1[df1['digits'] == 7].plot(kind='scatter', x='x', y='y', color='pink', label='7', ax=ax)
df1[df1['digits'] == 8].plot(kind='scatter', x='x', y='y', color='hotpink', label='8', ax=ax, figsize=(10,8), grid=False, title='Original dataset visualized by PCA')









    Out[32]:





<matplotlib.axes._subplots.AxesSubplot at 0x11c3857d0>



In [13]:

    
# reshape for convolutions
trainX = trainX.reshape((trainX.shape[0], 1, 28, 28))
testX = testX.reshape((testX.shape[0], 1, 28, 28))
valX = valX.reshape((valX.shape[0], 1, 28, 28))

# convert class vectors to binary class matrices
trainY = np_utils.to_categorical(trainY, nb_classes)
testY = np_utils.to_categorical(testY, nb_classes)
valY = np_utils.to_categorical(valY, nb_classes)



In [14]:

    
# Creating the model which consists of 3 conv layers followed by
# 2 fully conntected layers

# Sequential wrapper model
model = Sequential()

# first convolutional layer
model.add(Convolution2D(32, 1, 2, 2))
model.add(Activation('relu'))

# second convolutional layer
model.add(Convolution2D(48, 32, 2, 2))
model.add(Activation('relu')) 
model.add(MaxPooling2D(poolsize=(2,2)))

# third convolutional layer
model.add(Convolution2D(32, 48, 2, 2))
model.add(Activation('relu'))
model.add(MaxPooling2D(poolsize=(2,2)))

# convert convolutional filters to flatt so they can be feed to 
# fully connected layers
model.add(Flatten())

# first fully connected layer
model.add(Dense(32*6*6, 144, init='lecun_uniform'))
model.add(Activation('relu'))
model.add(Dropout(0.5))

# second fully connected layer
model.add(Dense(144, 144, init='lecun_uniform'))
model.add(Activation('relu'))
model.add(Dropout(0.5))

# last fully connected layer which output classes
model.add(Dense(144, 9, init='lecun_uniform'))
model.add(Activation('softmax'))

# setting sgd optimizer parameters
sgd = SGD(lr=0.05, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy', optimizer=sgd)

model.fit(trainX, trainY,
          batch_size=batch_size,
          nb_epoch=nb_epoch,
          show_accuracy=True,
          verbose=1,
          validation_data=(testX, testY))









    



Train on 3600 samples, validate on 1200 samples
Epoch 0
3600/3600 [==============================] - 154s - loss: 2.1808 - acc.: 0.1514 - val. loss: 2.1652 - val. acc.: 0.1606
Epoch 1
3600/3600 [==============================] - 148s - loss: 2.1814 - acc.: 0.1736 - val. loss: 2.1967 - val. acc.: 0.0984
Epoch 2
3600/3600 [==============================] - 143s - loss: 2.0944 - acc.: 0.2233 - val. loss: 1.2848 - val. acc.: 0.5724
Epoch 3
3600/3600 [==============================] - 149s - loss: 0.7510 - acc.: 0.7886 - val. loss: 0.3547 - val. acc.: 0.9117






    Out[14]:





{'acc': [0.15138888888888888,
  0.1736111111111111,
  0.22333333333333333,
  0.78861111111111115],
 'epoch': [0, 1, 2, 3],
 'loss': [2.180788974507502,
  2.1813934684519611,
  2.0943802670057741,
  0.7510084913201055],
 'val_acc': [0.16063596491228072,
  0.09841008771929825,
  0.5723684210526315,
  0.9117324561403509],
 'val_loss': [2.165183067764385,
  2.1967249362232852,
  1.284759149140705,
  0.35466280862711563]}



In [8]:

    
model.layers









    Out[8]:





[<keras.layers.convolutional.Convolution2D at 0x10a0c4850>,
 <keras.layers.core.Activation at 0x104505050>,
 <keras.layers.convolutional.Convolution2D at 0x104505850>,
 <keras.layers.core.Activation at 0x10f1c9510>,
 <keras.layers.convolutional.MaxPooling2D at 0x10f1c9810>,
 <keras.layers.convolutional.Convolution2D at 0x10f1c9610>,
 <keras.layers.core.Activation at 0x10f1c9c10>,
 <keras.layers.convolutional.MaxPooling2D at 0x10f1c9e90>,
 <keras.layers.core.Flatten at 0x10f1c9c90>,
 <keras.layers.core.Dense at 0x10f1c9fd0>,
 <keras.layers.core.Activation at 0x10f1c9ed0>,
 <keras.layers.core.Dropout at 0x10f1d4250>,
 <keras.layers.core.Dense at 0x10f1d42d0>,
 <keras.layers.core.Activation at 0x10f1d4090>,
 <keras.layers.core.Dropout at 0x10f1d4550>,
 <keras.layers.core.Dense at 0x10f1d45d0>,
 <keras.layers.core.Activation at 0x10f1d4350>]



In [9]:

    
for l in range(len(model.layers)):
    if model.layers[l].get_weights() != []:
        print l
        print model.layers[l]
#         print model.layers[l].get_weights()









    



0
<keras.layers.convolutional.Convolution2D object at 0x10a0c4850>
2
<keras.layers.convolutional.Convolution2D object at 0x104505850>
5
<keras.layers.convolutional.Convolution2D object at 0x10f1c9610>
9
<keras.layers.core.Dense object at 0x10f1c9fd0>
12
<keras.layers.core.Dense object at 0x10f1d42d0>
15
<keras.layers.core.Dense object at 0x10f1d45d0>



In [15]:

    
# Sequential wrapper model
model1 = Sequential()

# first convolutional layer
model1.add(Convolution2D(32, 1, 2, 2, weights=model.layers[0].get_weights()))
model1.add(Activation('relu'))

# second convolutional layer
model1.add(Convolution2D(48, 32, 2, 2, weights=model.layers[2].get_weights()))
model1.add(Activation('relu')) 
model1.add(MaxPooling2D(poolsize=(2,2)))

# third convolutional layer
model1.add(Convolution2D(32, 48, 2, 2, weights=model.layers[5].get_weights()))
model1.add(Activation('relu'))
model1.add(MaxPooling2D(poolsize=(2,2)))

# convert convolutional filters to flatt so they can be feed to 
# fully connected layers
model1.add(Flatten())

# first fully connected layer
model1.add(Dense(32*6*6, 144, init='lecun_uniform', weights=model.layers[9].get_weights()))
model1.add(Activation('relu'))
model1.add(Dropout(0.5))

model1.compile(loss='categorical_crossentropy', optimizer=sgd)



In [25]:

    
valX_pca = model1.predict(valX)









    



1200/1200 [==============================] - 12s



In [36]:

    
pca = RandomizedPCA(n_components=2)
valX_pca = pca.fit_transform(valX_pca)

df2 = pd.DataFrame(valX_pca)
df2['digits'] = np_utils.probas_to_classes(valY)
df2.columns = ['x', 'y', 'digits']

ax = df2[df2['digits'] == 0].plot(kind='scatter', x='x', y='y', color='blue', label='0')
ax = df2[df2['digits'] == 1].plot(kind='scatter', x='x', y='y', color='green', label='1', ax=ax)
ax = df2[df2['digits'] == 2].plot(kind='scatter', x='x', y='y', color='red', label='2', ax=ax)
ax = df2[df2['digits'] == 3].plot(kind='scatter', x='x', y='y', color='cyan', label='3', ax=ax)
ax = df2[df2['digits'] == 4].plot(kind='scatter', x='x', y='y', color='magenta', label='4', ax=ax)
ax = df2[df2['digits'] == 5].plot(kind='scatter', x='x', y='y', color='yellow', label='5', ax=ax)
ax = df2[df2['digits'] == 6].plot(kind='scatter', x='x', y='y', color='black', label='6', ax=ax)
ax = df2[df2['digits'] == 7].plot(kind='scatter', x='x', y='y', color='pink', label='7', ax=ax)
df2[df2['digits'] == 8].plot(kind='scatter', x='x', y='y', color='hotpink', label='8', ax=ax, figsize=(10,8), grid=False, title='first fully connected layer of dataset visualized by PCA')









    Out[36]:





<matplotlib.axes._subplots.AxesSubplot at 0x11cde2210>



In [37]:

    
valX_pca = model.predict(valX)









    



1200/1200 [==============================] - 12s



In [38]:

    
pca = RandomizedPCA(n_components=2)
valX_pca = pca.fit_transform(valX_pca)

df3 = pd.DataFrame(valX_pca)
df3['digits'] = np_utils.probas_to_classes(valY)
df3.columns = ['x', 'y', 'digits']

ax = df3[df3['digits'] == 0].plot(kind='scatter', x='x', y='y', color='blue', label='0')
ax = df3[df3['digits'] == 1].plot(kind='scatter', x='x', y='y', color='green', label='1', ax=ax)
ax = df3[df3['digits'] == 2].plot(kind='scatter', x='x', y='y', color='red', label='2', ax=ax)
ax = df3[df3['digits'] == 3].plot(kind='scatter', x='x', y='y', color='cyan', label='3', ax=ax)
ax = df3[df3['digits'] == 4].plot(kind='scatter', x='x', y='y', color='magenta', label='4', ax=ax)
ax = df3[df3['digits'] == 5].plot(kind='scatter', x='x', y='y', color='yellow', label='5', ax=ax)
ax = df3[df3['digits'] == 6].plot(kind='scatter', x='x', y='y', color='black', label='6', ax=ax)
ax = df3[df3['digits'] == 7].plot(kind='scatter', x='x', y='y', color='pink', label='7', ax=ax)
df3[df3['digits'] == 8].plot(kind='scatter', x='x', y='y', color='hotpink', label='8', ax=ax, figsize=(10,8), grid=False, title='last fully connected layer of dataset visualized by PCA')









    Out[38]:





<matplotlib.axes._subplots.AxesSubplot at 0x11d054190>

	x	y	digits
0	-2.811651	-4.363844	2
1	2.048056	7.475353	4
2	-2.290152	-3.085424	2
3	-2.234326	-2.303282	2
4	6.034777	-1.733684	8