In [1]:

    

from keras.datasets import mnist
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation
from keras.regularizers import l2, l1
from keras.constraints import maxnorm
from keras.optimizers import SGD, Adam, RMSprop
from keras.utils import np_utils
import numpy as np
import pandas as pd
from sklearn.datasets import fetch_mldata
from sklearn.cross_validation import train_test_split
from sklearn.datasets import fetch_mldata
from sklearn.metrics import classification_report
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
from time import time


    

In [2]:

    

batch_size = 100
nb_classes = 10
nb_epoch = 2
np.random.seed(1337) # for reproducibility


    

In [3]:

    

mnist = fetch_mldata('mnist-original')
X_train, X_test, y_train, y_test = train_test_split(
        (mnist.data / 255.0).astype(np.float32),
        mnist.target.astype(np.int32),
        test_size=0.33, random_state=1234)


    

In [4]:

    

# convert class vectors to binary class matrices
y_train = np_utils.to_categorical(y_train, nb_classes)
y_test = np_utils.to_categorical(y_test, nb_classes)


    

In [11]:

    

model = Sequential()
model.add(Dense(784, 128))
model.add(Activation('relu'))
model.add(Dense(128, 128))
model.add(Activation('relu'))
model.add(Dense(128, 10))
model.add(Activation('softmax'))

rms = RMSprop()
model.compile(loss='categorical_crossentropy', optimizer=rms)


    

In [12]:

    

model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=True, verbose=2)


    

    




Epoch 0
2s - loss: 0.4156 - acc.: 0.8825
Epoch 1
2s - loss: 0.1750 - acc.: 0.9482






    
Out[12]:





{'acc': [0.88245202558635394, 0.9482089552238806],
 'epoch': [0, 1],
 'loss': [0.41560721486208752, 0.17501242122249899]}

In [41]:

    

model2 = Sequential()
model2.add(Dense(784, 128, weights=model.layers[0].get_weights()))
model2.add(Activation('relu'))

model2.compile(loss='categorical_crossentropy', optimizer=rms)


    

In [43]:

    

a = model2.predict(X_test)


    

    




23100/23100 [==============================] - 0s     

In [49]:

    

print a[0].shape
print y_test[0]


    

    




(128,)
[ 0.  0.  0.  0.  1.  0.  0.  0.  0.  0.]

In [51]:

    

28*28


    

    
Out[51]:





784

In [ ]:

    




    

In [30]:

    

print model.layers[0].get_weights()[0].shape
print model.layers[0].get_weights()[1].shape


    

    




(784, 128)
(128,)

In [38]:

    

model.predict_proba(X_test)


    

    




23100/23100 [==============================] - 0s     






    
Out[38]:





array([[  1.50312530e-05,   1.76192264e-07,   8.22625685e-05, ...,
          1.95412379e-05,   3.72427236e-05,   5.16098000e-04],
       [  4.55915244e-06,   8.40894877e-07,   5.37658340e-06, ...,
          5.34605132e-04,   4.70654030e-05,   9.79131553e-01],
       [  1.09445249e-03,   1.71516844e-05,   1.86267585e-03, ...,
          3.49014721e-04,   3.74491535e-05,   1.10402169e-03],
       ..., 
       [  5.06238162e-04,   2.16803142e-05,   9.80179968e-01, ...,
          3.22616644e-04,   3.09785075e-03,   1.01201899e-05],
       [  5.53817717e-06,   2.19545802e-05,   1.97015418e-05, ...,
          3.97852334e-05,   3.03472929e-02,   1.86204362e-02],
       [  2.96912506e-05,   4.44483166e-04,   5.09012152e-05, ...,
          8.78157401e-05,   4.88553201e-02,   7.53322212e-04]])

In [39]:

    

model.predict_classes(X_test)


    

    




23100/23100 [==============================] - 0s     






    
Out[39]:





array([4, 9, 6, ..., 2, 3, 5])

In [40]:

    

model.predict(X_test)


    

    




23100/23100 [==============================] - 0s     






    
Out[40]:





array([[  1.50312530e-05,   1.76192264e-07,   8.22625685e-05, ...,
          1.95412379e-05,   3.72427236e-05,   5.16098000e-04],
       [  4.55915244e-06,   8.40894877e-07,   5.37658340e-06, ...,
          5.34605132e-04,   4.70654030e-05,   9.79131553e-01],
       [  1.09445249e-03,   1.71516844e-05,   1.86267585e-03, ...,
          3.49014721e-04,   3.74491535e-05,   1.10402169e-03],
       ..., 
       [  5.06238162e-04,   2.16803142e-05,   9.80179968e-01, ...,
          3.22616644e-04,   3.09785075e-03,   1.01201899e-05],
       [  5.53817717e-06,   2.19545802e-05,   1.97015418e-05, ...,
          3.97852334e-05,   3.03472929e-02,   1.86204362e-02],
       [  2.96912506e-05,   4.44483166e-04,   5.09012152e-05, ...,
          8.78157401e-05,   4.88553201e-02,   7.53322212e-04]])

In [7]:

    

print model.get_weights()[0].shape
print model.get_weights()[1].shape
print model.get_weights()[2].shape
print model.get_weights()[3].shape
print model.get_weights()[4].shape
print model.get_weights()[5].shape


    

    




(784, 128)
(128,)
(128, 128)
(128,)
(128, 10)
(10,)

In [8]:

    

print model.get_output()


    

    




Softmax.0

In [9]:

    

model.predict_proba


    

    
Out[9]:





<bound method Sequential.predict_proba of <keras.models.Sequential object at 0x109ba00d0>>

In [10]:

    

print model.layers[0]
print model.layers[1]
l1 = model.layers[0]
l2 = model.layers[1]


    

    




<keras.layers.core.Dense object at 0x109ba0090>
<keras.layers.core.Activation object at 0x109ba0150>

In [11]:

    

for l in model.layers:
    ww = l.get_weights()
    for i in ww:
        print i.shape
    print


    

    




(784, 128)
(128,)



(128, 128)
(128,)



(128, 10)
(10,)

In [12]:

    

print l1.W
print l1.activation
print l1.b
print l1.connect
print l1.constraints
print l1.get_config()
print l1.get_input
print l1.init
print l1.input
print l1.get_weights()
print l1.input_dim
print l1.output_dim
print l1.params
print l1.regularizers
print l1.set_weights


    

    




<TensorType(float64, matrix)>
<function linear at 0x107206aa0>
<TensorType(float64, vector)>
<bound method Dense.connect of <keras.layers.core.Dense object at 0x109ba0090>>
[None, None]
{'output_dim': 128, 'init': 'glorot_uniform', 'activation': 'linear', 'name': 'Dense', 'input_dim': 784}
<bound method Dense.get_input of <keras.layers.core.Dense object at 0x109ba0090>>
<function glorot_uniform at 0x107206e60>
<TensorType(float64, matrix)>
[array([[-0.02228843, -0.03196717, -0.02078036, ...,  0.01706213,
        -0.00175904,  0.04088276],
       [ 0.00762368, -0.02939873, -0.02932186, ..., -0.03431994,
         0.02931596,  0.02220792],
       [ 0.02288999,  0.03217546,  0.02339208, ..., -0.03766216,
        -0.02539823, -0.03341346],
       ..., 
       [ 0.00722906, -0.02683882, -0.02172514, ..., -0.03012692,
        -0.00718376, -0.00656352],
       [-0.00882501, -0.03115064, -0.01856273, ..., -0.01167483,
         0.03620235,  0.03301103],
       [ 0.03888712, -0.03137559, -0.0374804 , ..., -0.03370819,
         0.03950158, -0.00325403]]), array([-0.04393619, -0.00422617,  0.04508981,  0.01635258,  0.03636349,
        0.02634787,  0.03253269, -0.0068647 , -0.06086822,  0.08127944,
        0.01407352,  0.07477905,  0.00250433,  0.02977351,  0.01453377,
       -0.02660448,  0.02751523, -0.0107354 ,  0.02040303, -0.02865396,
        0.06438286,  0.00011104,  0.05717212,  0.05657974, -0.00596116,
       -0.08379902,  0.00780386,  0.03357766,  0.04086832, -0.02184547,
        0.03747951,  0.01992137,  0.02316762,  0.00906717, -0.07557154,
        0.05468904, -0.04335818,  0.04930746, -0.03245607,  0.05579761,
       -0.02487587,  0.07829946, -0.07471236,  0.06605285, -0.01115548,
       -0.01530014, -0.00527784, -0.04731319,  0.10676087,  0.06106861,
       -0.00945507,  0.01844439, -0.0182679 ,  0.04433949,  0.04059699,
        0.10093575, -0.00392908,  0.00769327,  0.04665201, -0.0638851 ,
       -0.04787099,  0.04686063,  0.02885133,  0.03489821,  0.03662819,
        0.00573642,  0.07087084,  0.06800933,  0.05542223,  0.01200268,
       -0.01617728,  0.04775283,  0.00684704,  0.06892321, -0.0265125 ,
        0.05120812, -0.01531227, -0.04826067, -0.01503271, -0.09236347,
       -0.04388734, -0.00299919,  0.05741286,  0.02983764,  0.05269021,
       -0.02274182,  0.07784502, -0.01525284,  0.07082865,  0.02331247,
        0.05026318, -0.01514979,  0.00384451,  0.05296629,  0.0102998 ,
        0.05839575,  0.08244099,  0.05154108, -0.08377316, -0.01328637,
        0.09106039, -0.04377876,  0.0089555 ,  0.00844728,  0.00452837,
        0.0242163 ,  0.08078732, -0.01311797,  0.07030779,  0.01277792,
       -0.07008811,  0.02463853, -0.04300062, -0.03774756,  0.09036096,
        0.00775428, -0.00919777,  0.07023773,  0.0331726 ,  0.073409  ,
       -0.01337602, -0.01082848,  0.00539046, -0.00897375, -0.02152097,
       -0.06372975,  0.05370549,  0.00520219])]
784
128
[<TensorType(float64, matrix)>, <TensorType(float64, vector)>]
[None, None]
<bound method Dense.set_weights of <keras.layers.core.Dense object at 0x109ba0090>>

In [13]:

    

print l1.activation
print l2.activation
print l1.get_weights()
print l2.get_weights()


    

    




<function linear at 0x107206aa0>
<function relu at 0x1072068c0>
[array([[-0.02228843, -0.03196717, -0.02078036, ...,  0.01706213,
        -0.00175904,  0.04088276],
       [ 0.00762368, -0.02939873, -0.02932186, ..., -0.03431994,
         0.02931596,  0.02220792],
       [ 0.02288999,  0.03217546,  0.02339208, ..., -0.03766216,
        -0.02539823, -0.03341346],
       ..., 
       [ 0.00722906, -0.02683882, -0.02172514, ..., -0.03012692,
        -0.00718376, -0.00656352],
       [-0.00882501, -0.03115064, -0.01856273, ..., -0.01167483,
         0.03620235,  0.03301103],
       [ 0.03888712, -0.03137559, -0.0374804 , ..., -0.03370819,
         0.03950158, -0.00325403]]), array([-0.04393619, -0.00422617,  0.04508981,  0.01635258,  0.03636349,
        0.02634787,  0.03253269, -0.0068647 , -0.06086822,  0.08127944,
        0.01407352,  0.07477905,  0.00250433,  0.02977351,  0.01453377,
       -0.02660448,  0.02751523, -0.0107354 ,  0.02040303, -0.02865396,
        0.06438286,  0.00011104,  0.05717212,  0.05657974, -0.00596116,
       -0.08379902,  0.00780386,  0.03357766,  0.04086832, -0.02184547,
        0.03747951,  0.01992137,  0.02316762,  0.00906717, -0.07557154,
        0.05468904, -0.04335818,  0.04930746, -0.03245607,  0.05579761,
       -0.02487587,  0.07829946, -0.07471236,  0.06605285, -0.01115548,
       -0.01530014, -0.00527784, -0.04731319,  0.10676087,  0.06106861,
       -0.00945507,  0.01844439, -0.0182679 ,  0.04433949,  0.04059699,
        0.10093575, -0.00392908,  0.00769327,  0.04665201, -0.0638851 ,
       -0.04787099,  0.04686063,  0.02885133,  0.03489821,  0.03662819,
        0.00573642,  0.07087084,  0.06800933,  0.05542223,  0.01200268,
       -0.01617728,  0.04775283,  0.00684704,  0.06892321, -0.0265125 ,
        0.05120812, -0.01531227, -0.04826067, -0.01503271, -0.09236347,
       -0.04388734, -0.00299919,  0.05741286,  0.02983764,  0.05269021,
       -0.02274182,  0.07784502, -0.01525284,  0.07082865,  0.02331247,
        0.05026318, -0.01514979,  0.00384451,  0.05296629,  0.0102998 ,
        0.05839575,  0.08244099,  0.05154108, -0.08377316, -0.01328637,
        0.09106039, -0.04377876,  0.0089555 ,  0.00844728,  0.00452837,
        0.0242163 ,  0.08078732, -0.01311797,  0.07030779,  0.01277792,
       -0.07008811,  0.02463853, -0.04300062, -0.03774756,  0.09036096,
        0.00775428, -0.00919777,  0.07023773,  0.0331726 ,  0.073409  ,
       -0.01337602, -0.01082848,  0.00539046, -0.00897375, -0.02152097,
       -0.06372975,  0.05370549,  0.00520219])]
[]

In [14]:

    

tmp1 = l1.get_input(X_train[0])
tmp2 = l1.get_output(X_train[0])


    

In [39]:

    

l2.previous.get_output


    

    
Out[39]:





<bound method Dense.get_output of <keras.layers.core.Dense object at 0x109ba0090>>

In [29]:

    

print l1.get_input(X_train)


    

    




<TensorType(float64, matrix)>

In [27]:

    

print l1.get_output(l1.get_input(X_train))


    

    




Elemwise{add,no_inplace}.0

In [40]:

    

X_train[0:2].shape


    

    
Out[40]:





(2, 784)

In [42]:

    

a1 = l1.get_input(X_train)


    

In [44]:

    

a2 = l1.get_output(a1)


    

In [52]:

    

a3 = l2.get_input(a2)


    

In [62]:

    

print l2.previous
print l2.get_input(l2.previous)
a = l2.get_output(l2.get_input(l2.previous))


    

    




<keras.layers.core.Dense object at 0x109ba0090>
Elemwise{add,no_inplace}.0

In [67]:

    

a.shape


    

    
Out[67]:





Shape.0