In [54]:

    

from sklearn.datasets import load_digits
X,y = load_digits().data,load_digits().target
from sklearn.decomposition import PCA
X = PCA(20).fit_transform(X)
X.shape


    

    
Out[54]:





(1797, 20)

In [55]:

    

X_train,X_test = X[:1000],X[1000:]
y_train,y_test = Y[:1000],Y[1000:]


    

In [56]:

    

len(set(y_test))


    

    
Out[56]:





10

In [57]:

    

from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation
from keras.optimizers import SGD

model = Sequential()
model.add(Dense(20, 64, init='uniform',activation='tanh'))
model.add(Dropout(0.5))
model.add(Dense(64, 64, init='uniform',activation='tanh'))
model.add(Dropout(0.5))
model.add(Dense(64, 10, init='uniform',activation='softmax'))

sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='mean_squared_error', optimizer=sgd)

model.fit(X_train, y_train, nb_epoch=20, batch_size=16)
score = model.evaluate(X_test, y_test, batch_size=16)


    

    




Epoch 0






    




---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-57-90f0d4674041> in <module>()
     13 model.compile(loss='mean_squared_error', optimizer=sgd)
     14 
---> 15 model.fit(X_train, y_train, nb_epoch=20, batch_size=16)
     16 score = model.evaluate(X_test, y_test, batch_size=16)

/Users/joshuafass/anaconda/envs/py27/lib/python2.7/site-packages/Keras-0.0.1-py2.7.egg/keras/models.pyc in fit(self, X, y, batch_size, nb_epoch, verbose, validation_split, validation_data, shuffle, show_accuracy)
    250                     av_acc += acc * len(batch_ids)
    251                 else:
--> 252                     loss = self._train(*ins)
    253                     log_values = [('loss', loss)]
    254                     av_loss += loss * len(batch_ids)

/Users/joshuafass/anaconda/envs/py27/lib/python2.7/site-packages/theano/compile/function_module.pyc in __call__(self, *args, **kwargs)
    604                         self.fn.nodes[self.fn.position_of_error],
    605                         self.fn.thunks[self.fn.position_of_error],
--> 606                         storage_map=self.fn.storage_map)
    607                 else:
    608                     # For the c linker We don't have access from

/Users/joshuafass/anaconda/envs/py27/lib/python2.7/site-packages/theano/compile/function_module.pyc in __call__(self, *args, **kwargs)
    593         t0_fn = time.time()
    594         try:
--> 595             outputs = self.fn()
    596         except Exception:
    597             if hasattr(self.fn, 'position_of_error'):

ValueError: Input dimension mis-match. (input[0].shape[1] = 10, input[1].shape[1] = 1)
Apply node that caused the error: Elemwise{sub,no_inplace}(SoftmaxWithBias.0, <TensorType(float32, matrix)>)
Inputs types: [TensorType(float32, matrix), TensorType(float32, matrix)]
Inputs shapes: [(16, 10), (16, 1)]
Inputs strides: [(40, 4), (4, 4)]
Inputs values: ['not shown', 'not shown']

Backtrace when the node is created:
  File "build/bdist.macosx-10.5-x86_64/egg/keras/objectives.py", line 10, in mean_squared_error
    return T.sqr(y_pred - y_true).mean()

HINT: Use the Theano flag 'exception_verbosity=high' for a debugprint and storage map footprint of this apply node.

In [58]:

    

X.shape


    

    
Out[58]:





(1797, 20)

In [59]:

    

# stolen from: https://github.com/fchollet/keras/blob/master/examples/mnist_nn.py

from __future__ import absolute_import
from __future__ import print_function
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation
from keras.regularizers import l2, l1
from keras.constraints import maxnorm
from keras.optimizers import SGD, Adam, RMSprop
from keras.utils import np_utils
import numpy as np

'''
    Train a simple deep NN on the MNIST dataset.
'''

batch_size = 64
nb_classes = 10
nb_epoch = 20

np.random.seed(1337) # for reproducibility

# the data, shuffled and split between tran and test sets
(X_train, y_train), (X_test, y_test) = mnist.load_data()

X_train=X_train.reshape(60000,784)
X_test=X_test.reshape(10000,784)
X_train = X_train.astype("float32")
X_test = X_test.astype("float32")
X_train /= 255
X_test /= 255
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')

# convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)

model = Sequential()
model.add(Dense(784, 128))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(128, 128))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(128, 10))
model.add(Activation('softmax'))

rms = RMSprop()
model.compile(loss='categorical_crossentropy', optimizer=rms)

model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=True, verbose=2, validation_data=(X_test, Y_test))
score = model.evaluate(X_test, Y_test, show_accuracy=True, verbose=0)
print('Test score:', score[0])
print('Test accuracy:', score[1])


    

    




60000 train samples
10000 test samples
Train on 60000 samples, validate on 10000 samples
Epoch 0
2s - loss: 0.4002 - acc.: 0.8834 - val. loss: 0.1680 - val. acc.: 0.9499
Epoch 1
2s - loss: 0.1756 - acc.: 0.9474 - val. loss: 0.1189 - val. acc.: 0.9653
Epoch 2
2s - loss: 0.1322 - acc.: 0.9602 - val. loss: 0.0984 - val. acc.: 0.9701
Epoch 3
2s - loss: 0.1095 - acc.: 0.9675 - val. loss: 0.0817 - val. acc.: 0.9749
Epoch 4
2s - loss: 0.0960 - acc.: 0.9711 - val. loss: 0.0795 - val. acc.: 0.9758
Epoch 5
2s - loss: 0.0844 - acc.: 0.9737 - val. loss: 0.0863 - val. acc.: 0.9766
Epoch 6
2s - loss: 0.0775 - acc.: 0.9764 - val. loss: 0.0747 - val. acc.: 0.9789
Epoch 7
2s - loss: 0.0746 - acc.: 0.9769 - val. loss: 0.0721 - val. acc.: 0.9780
Epoch 8
2s - loss: 0.0669 - acc.: 0.9792 - val. loss: 0.0721 - val. acc.: 0.9817
Epoch 9
2s - loss: 0.0627 - acc.: 0.9805 - val. loss: 0.0697 - val. acc.: 0.9806
Epoch 10
2s - loss: 0.0602 - acc.: 0.9816 - val. loss: 0.0740 - val. acc.: 0.9795
Epoch 11
2s - loss: 0.0578 - acc.: 0.9815 - val. loss: 0.0811 - val. acc.: 0.9788
Epoch 12
2s - loss: 0.0563 - acc.: 0.9832 - val. loss: 0.0739 - val. acc.: 0.9800
Epoch 13
2s - loss: 0.0520 - acc.: 0.9844 - val. loss: 0.0795 - val. acc.: 0.9799
Epoch 14
2s - loss: 0.0493 - acc.: 0.9845 - val. loss: 0.0699 - val. acc.: 0.9825
Epoch 15
2s - loss: 0.0483 - acc.: 0.9851 - val. loss: 0.0760 - val. acc.: 0.9820
Epoch 16
3s - loss: 0.0472 - acc.: 0.9853 - val. loss: 0.0775 - val. acc.: 0.9809
Epoch 17
2s - loss: 0.0457 - acc.: 0.9859 - val. loss: 0.0742 - val. acc.: 0.9800
Epoch 18
2s - loss: 0.0421 - acc.: 0.9871 - val. loss: 0.0781 - val. acc.: 0.9825
Epoch 19
2s - loss: 0.0424 - acc.: 0.9866 - val. loss: 0.0759 - val. acc.: 0.9823
Test score: 0.0750831614406
Test accuracy: 0.982496044304

In [26]:

    

# let's make an autoencoder

model = Sequential()
model.add(Dense(784, 128,activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(128, 128,activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(128, 2,activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(2, 128,activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(128, 128,activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(128, 784,activation='relu'))

rms = RMSprop()
model.compile(loss='mse', optimizer=rms)

model.fit(X_train, X_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=True, verbose=2, validation_data=(X_test, X_test))
score = model.evaluate(X_test, X_test, show_accuracy=True, verbose=0)
print('Test score:', score[0])
print('Test accuracy:', score[1])


    

    




Train on 60000 samples, validate on 10000 samples
Epoch 0
5s - loss: 0.0758 - acc.: 0.0088 - val. loss: 0.0676 - val. acc.: 0.0143
Epoch 1
5s - loss: 0.0672 - acc.: 0.0104 - val. loss: 0.0657 - val. acc.: 0.0130
Epoch 2
5s - loss: 0.0656 - acc.: 0.0114 - val. loss: 0.0637 - val. acc.: 0.0126
Epoch 3
5s - loss: 0.0644 - acc.: 0.0115 - val. loss: 0.0625 - val. acc.: 0.0105
Epoch 4
5s - loss: 0.0638 - acc.: 0.0123 - val. loss: 0.0621 - val. acc.: 0.0124
Epoch 5
5s - loss: 0.0636 - acc.: 0.0123 - val. loss: 0.0620 - val. acc.: 0.0132
Epoch 6
5s - loss: 0.0634 - acc.: 0.0129 - val. loss: 0.0618 - val. acc.: 0.0134
Epoch 7
5s - loss: 0.0633 - acc.: 0.0122 - val. loss: 0.0618 - val. acc.: 0.0134
Epoch 8
5s - loss: 0.0633 - acc.: 0.0130 - val. loss: 0.0618 - val. acc.: 0.0133
Epoch 9
5s - loss: 0.0632 - acc.: 0.0134 - val. loss: 0.0616 - val. acc.: 0.0136
Epoch 10
5s - loss: 0.0631 - acc.: 0.0132 - val. loss: 0.0616 - val. acc.: 0.0139
Epoch 11
5s - loss: 0.0629 - acc.: 0.0135 - val. loss: 0.0615 - val. acc.: 0.0138
Epoch 12
5s - loss: 0.0628 - acc.: 0.0129 - val. loss: 0.0612 - val. acc.: 0.0137
Epoch 13
5s - loss: 0.0622 - acc.: 0.0136 - val. loss: 0.0605 - val. acc.: 0.0141
Epoch 14
5s - loss: 0.0612 - acc.: 0.0125 - val. loss: 0.0603 - val. acc.: 0.0139
Epoch 15
5s - loss: 0.0608 - acc.: 0.0119 - val. loss: 0.0600 - val. acc.: 0.0133
Epoch 16
5s - loss: 0.0606 - acc.: 0.0107 - val. loss: 0.0599 - val. acc.: 0.0136
Epoch 17
5s - loss: 0.0605 - acc.: 0.0103 - val. loss: 0.0598 - val. acc.: 0.0126
Epoch 18
5s - loss: 0.0604 - acc.: 0.0096 - val. loss: 0.0595 - val. acc.: 0.0131
Epoch 19
5s - loss: 0.0602 - acc.: 0.0096 - val. loss: 0.0594 - val. acc.: 0.0095
Test score: 0.0594051815569
Test accuracy: 0.00939477848101

In [23]:

    

X_train.shape


    

    
Out[23]:





(60000, 784)

In [25]:

    

model.predict(X_train).shape


    

    




60000/60000 [==============================] - 1s     






    
Out[25]:





(60000, 784)

In [33]:

    

model.get_weights()[:3]


    

    
Out[33]:





[array([[ 0.0181485 ,  0.02609826, -0.00305522, ..., -0.00780296,
          0.02137184,  0.01811046],
        [ 0.02537466, -0.04157665,  0.02020432, ...,  0.04484425,
         -0.00866545,  0.01702437],
        [ 0.02393472,  0.02866936, -0.00472653, ...,  0.01880961,
          0.00587549,  0.02051604],
        ..., 
        [-0.00808461,  0.0281772 , -0.03982936, ...,  0.03785738,
         -0.00189981,  0.03640595],
        [-0.01715138,  0.00567307,  0.04657373, ...,  0.03418556,
         -0.01153133, -0.02722255],
        [ 0.00566882,  0.01680367, -0.01889592, ...,  0.00166049,
          0.03052141,  0.03853069]], dtype=float32),
 array([-0.07241112,  0.0069766 ,  0.00392984,  0.03992437, -0.08026286,
         0.00296543, -0.07084998,  0.01232044,  0.01584496, -0.00962455,
         0.01415984,  0.0069422 , -0.00905515, -0.00570969, -0.00244865,
         0.01633407, -0.08068327,  0.00753555, -0.09376856, -0.10046797,
        -0.03396196, -0.08234026,  0.04115795,  0.01795877, -0.09049919,
         0.01218412, -0.10286569,  0.01023567,  0.00714153,  0.00988461,
         0.01545758, -0.00127244,  0.02261891, -0.00272064, -0.00262476,
        -0.05834663,  0.02771454, -0.070443  ,  0.00443574,  0.01952106,
        -0.0881573 , -0.02852612,  0.01938807,  0.00097267, -0.00574786,
        -0.02628584,  0.01840769, -0.00464533,  0.00096442,  0.00893583,
        -0.07917481,  0.00031921, -0.04541691, -0.10835235,  0.01179633,
         0.02184967, -0.00065211, -0.06013202,  0.01292379, -0.01975267,
        -0.07254777, -0.05152095, -0.0006332 , -0.03258261, -0.00475059,
        -0.03976971,  0.0241556 ,  0.00127159, -0.0028063 ,  0.00952145,
        -0.11381801, -0.01757858, -0.00192061, -0.06682976, -0.02527649,
         0.02709396, -0.00260095,  0.02078892, -0.00320256, -0.09860161,
        -0.04386458, -0.05868203,  0.04485738, -0.00871583, -0.00411345,
        -0.00262248,  0.01885691,  0.01184752, -0.11377472, -0.09902745,
        -0.07182229,  0.0523348 , -0.00273854,  0.05448765, -0.13177966,
        -0.07278015,  0.00576413, -0.04813857, -0.04759918,  0.01292087,
        -0.06375396,  0.00475342,  0.01710393, -0.0836885 ,  0.0032417 ,
        -0.04271514,  0.01347142,  0.0010778 ,  0.00189263, -0.00393991,
        -0.02015704,  0.00881423,  0.02087088,  0.02345028,  0.01245924,
        -0.00791272, -0.00290048,  0.03114179, -0.0795873 ,  0.0049227 ,
         0.00685469, -0.00741858,  0.00324824, -0.07507141,  0.05788776,
         0.0055315 , -0.00337293, -0.11769459], dtype=float32),
 array([[ 0.07986602, -0.08522494,  0.05265382, ..., -0.01025603,
         -0.00613539, -0.01853022],
        [ 0.01370056,  0.07116223, -0.02965094, ...,  0.0482785 ,
         -0.03991069, -0.0084573 ],
        [ 0.06689017, -0.02908228, -0.00272555, ..., -0.05713871,
          0.02182861,  0.04557681],
        ..., 
        [-0.00482133, -0.0644448 , -0.00851347, ..., -0.0460988 ,
          0.09085574, -0.07308838],
        [-0.08493724, -0.02241907,  0.04241071, ...,  0.04382087,
         -0.07029235, -0.027528  ],
        [ 0.01755759,  0.07159813, -0.0234807 , ..., -0.04688525,
          0.03097397, -0.06093774]], dtype=float32)]

In [36]:

    

import copy
model_layers_archive = copy.copy(model.layers)


    

In [37]:

    

model.layers = model_layers_archive[:6]
model.layers


    

    
Out[37]:





[<keras.layers.core.Dense at 0x11013f890>,
 <keras.layers.core.Dropout at 0x10e41add0>,
 <keras.layers.core.Dense at 0x10e4ba310>,
 <keras.layers.core.Dropout at 0x10d038590>,
 <keras.layers.core.Dense at 0x1104ba490>,
 <keras.layers.core.Dropout at 0x11093aa10>]

In [39]:

    

model.predict(X_train).shape


    

    




60000/60000 [==============================] - 5s     






    
Out[39]:





(60000, 784)

In [47]:

    

m = Sequential()
for l in model_layers_archive[:5]:
    m.add(l)


    

In [53]:

    

l = model_layers_archive[0]
l.get_output(X_train)


    

    
Out[53]:





Elemwise{true_div,no_inplace}.0

In [48]:

    

model.compile(loss='mse', optimizer=rms)


    

    




---------------------------------------------------------------------------
DisconnectedInputError                    Traceback (most recent call last)
<ipython-input-48-199376e9c9ce> in <module>()
----> 1 model.compile(loss='mse', optimizer=rms)

/Users/joshuafass/anaconda/envs/py27/lib/python2.7/site-packages/Keras-0.0.1-py2.7.egg/keras/models.pyc in compile(self, optimizer, loss, class_mode, theano_mode)
     92         self.class_mode = class_mode
     93 
---> 94         updates = self.optimizer.get_updates(self.params, self.regularizers, self.constraints, train_loss)
     95 
     96         if type(self.X_train) == list:

/Users/joshuafass/anaconda/envs/py27/lib/python2.7/site-packages/Keras-0.0.1-py2.7.egg/keras/optimizers.pyc in get_updates(self, params, regularizers, constraints, cost)
     68 
     69     def get_updates(self, params, regularizers, constraints, cost):
---> 70         grads = self.get_gradients(cost, params, regularizers)
     71         accumulators = [shared_zeros(p.get_value().shape) for p in params]
     72         updates = []

/Users/joshuafass/anaconda/envs/py27/lib/python2.7/site-packages/Keras-0.0.1-py2.7.egg/keras/optimizers.pyc in get_gradients(self, cost, params, regularizers)
     21 
     22     def get_gradients(self, cost, params, regularizers):
---> 23         grads = T.grad(cost, params)
     24 
     25         if hasattr(self, 'clipnorm') and self.clipnorm > 0:

/Users/joshuafass/anaconda/envs/py27/lib/python2.7/site-packages/theano/gradient.pyc in grad(cost, wrt, consider_constant, disconnected_inputs, add_names, known_grads, return_disconnected)
    527         if elem not in var_to_app_to_idx and elem is not cost \
    528                 and elem not in grad_dict:
--> 529             handle_disconnected(elem)
    530             grad_dict[elem] = disconnected_type()
    531 

/Users/joshuafass/anaconda/envs/py27/lib/python2.7/site-packages/theano/gradient.pyc in handle_disconnected(var)
    514                 warnings.warn(message, stacklevel=2)
    515             elif disconnected_inputs == 'raise':
--> 516                 raise DisconnectedInputError(message)
    517             else:
    518                 raise ValueError("Invalid value for keyword "

DisconnectedInputError: grad method was asked to compute the gradient with respect to a variable that is not part of the computational graph of the cost, or is used only by a non-differentiable operator: <TensorType(float32, matrix)>

In [45]:

    

m.predict(X_train)


    

    




---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-45-e3addf166985> in <module>()
----> 1 m.predict(X_train)

/Users/joshuafass/anaconda/envs/py27/lib/python2.7/site-packages/Keras-0.0.1-py2.7.egg/keras/models.pyc in predict(self, X, batch_size, verbose)
    284         for batch_index, (batch_start, batch_end) in enumerate(batches):
    285             X_batch = slice_X(X, batch_start, batch_end)
--> 286             batch_preds = self._predict(*X_batch)
    287 
    288             if batch_index == 0:

AttributeError: 'Sequential' object has no attribute '_predict'

In [ ]:

    

# let's do