In [1]:

    

import matplotlib.pyplot as plt
import numpy as np
from keras.layers import Input,Dense, Convolution2D, MaxPooling2D, UpSampling2D
from keras.models import Model
from keras.datasets import mnist
from keras import regularizers


    

    




Using Theano backend.

In [2]:

    

(x_train,_),(x_test,_) = mnist.load_data()

x_train = x_train.astype('float32') / 255
x_test = x_test.astype('float32') / 255
x_train = x_train.reshape((len(x_train), np.prod(x_train.shape[1:])))
x_test = x_test.reshape((len(x_test), np.prod(x_test.shape[1:])))


    

In [3]:

    

%matplotlib inline


    

In [4]:

    

def plot_imgs(decoded_images,n=20):

    plt.figure(figsize=(20,4))
    for i in range(n):
        # original digit
        ax = plt.subplot(2,n,i+1)
        plt.imshow(x_test[i].reshape(28,28))
        plt.gray()
        ax.get_xaxis().set_visible(False)
        ax.get_yaxis().set_visible(False)

        # reconstruction
        ax = plt.subplot(2,n,i+1+n)
        plt.imshow(decoded_images[i].reshape(28,28))
        plt.gray()
        ax.get_xaxis().set_visible(False)
        ax.get_yaxis().set_visible(False)
    
    plt.show()


    

In [ ]:

    

%%time

encoded = None
decoded = None

encoding_dim = 32
input_img = Input(shape=(784,))
encoded = Dense(encoding_dim, activation='relu')(input_img)
decoded = Dense(784,activation='sigmoid')(encoded)
autoencoder = Model(input=input_img, output=decoded)
encoder = Model(input=input_img, output = encoded)
encoded_input = Input(shape=(encoding_dim,))
decoder_layer = autoencoder.layers[-1]
decoder = Model(input=encoded_input, output=decoder_layer(encoded_input))
autoencoder.compile(optimizer='adadelta',loss='binary_crossentropy')


autoencoder.fit(x_train, x_train,
               verbose=2,  
               nb_epoch = 50,
               batch_size = 256,
               shuffle=True,
               validation_data=(x_test,x_test))

encoded_imgs = encoder.predict(x_test)
decoded_imgs = decoder.predict(encoded_imgs)

plot_imgs(decoded_imgs)


    

In [ ]:

    

encoded = None
decoded = None

encoding_dim = 32
input_img = Input(shape=(784,))

encoded = Dense(encoding_dim, activation='linear')(input_img)
decoded = Dense(784,activation='sigmoid')(encoded)
autoencoder = Model(input=input_img, output=decoded)

encoder = Model(input=input_img, output = encoded)
encoded_input = Input(shape=(encoding_dim,))


decoder_layer = autoencoder.layers[-1]
decoder = Model(input=encoded_input, output=decoder_layer(encoded_input))

autoencoder.compile(optimizer='adadelta',loss='binary_crossentropy')


autoencoder.fit(x_train, x_train,
               verbose=2,  
               nb_epoch = 50,
               batch_size = 256,
               shuffle=True,
               validation_data=(x_test,x_test))

encoded_imgs = encoder.predict(x_test)
decoded_imgs = decoder.predict(encoded_imgs)

plot_imgs(decoded_imgs)


    

In [ ]:

    

%%time

encoded = None
decoded = None

encoding_dim = 32

input_img = Input(shape=(784,))
# add a Dense layer with a L1 activity regularizer
encoded = Dense(encoding_dim, activation='relu',
                activity_regularizer=regularizers.activity_l1(10e-5))(input_img)
decoded = Dense(784, activation='sigmoid')(encoded)

autoencoder = Model(input=input_img, output=decoded)

encoder = Model(input=input_img, output = encoded)
encoded_input = Input(shape=(encoding_dim,))

decoder_layer = autoencoder.layers[-1]
decoder = Model(input=encoded_input, output=decoder_layer(encoded_input))
autoencoder.compile(optimizer='adadelta',loss='binary_crossentropy')


autoencoder.fit(x_train, x_train,
               verbose=2,  
               nb_epoch = 50,
               batch_size = 256,
               shuffle=True,
               validation_data=(x_test,x_test))

encoded_imgs = encoder.predict(x_test)
decoded_imgs = decoder.predict(encoded_imgs)

plot_imgs(decoded_imgs)


    

In [5]:

    

%%time

encoded = None
decoded = None

input_img = Input(shape=(784,))

encoded = Dense(128,activation='relu')(input_img)
encoded = Dense(64, activation='relu')(encoded)
encoded = Dense(32,activation='relu')(encoded)

decoded = Dense(64,activation='relu')(encoded)
decoded = Dense(128,activation='relu')(decoded)
decoded = Dense(784,activation='sigmoid')(decoded)

autoencoder = Model(input=input_img, output=decoded)


encoder = Model(input=input_img, output = encoded)
encoded_input = Input(shape=(32,))

decoder_layer1 = autoencoder.layers[-3]
decoder_layer2 = autoencoder.layers[-2]
decoder_layer3 = autoencoder.layers[-1]

decoder = Model(input=encoded_input, output= decoder_layer3(decoder_layer2(decoder_layer1(encoded_input))))

autoencoder.compile(optimizer='adadelta',loss='binary_crossentropy')

autoencoder.fit(x_train, x_train, nb_epoch=100, verbose=2,
               batch_size=256,shuffle=True,
               validation_data=(x_test, x_test))

encoded_imgs = encoder.predict(x_test)
decoded_imgs = decoder.predict(encoded_imgs)

plot_imgs(decoded_imgs)


    

    




Train on 60000 samples, validate on 10000 samples
Epoch 1/100
13s - loss: 0.3602 - val_loss: 0.2642
Epoch 2/100
13s - loss: 0.2599 - val_loss: 0.2558
Epoch 3/100
13s - loss: 0.2490 - val_loss: 0.2410
Epoch 4/100
13s - loss: 0.2341 - val_loss: 0.2223
Epoch 5/100






    




---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-5-8f401d38a93a> in <module>()
----> 1 get_ipython().run_cell_magic(u'time', u'', u"\nencoded = None\ndecoded = None\n\ninput_img = Input(shape=(784,))\n\nencoded = Dense(128,activation='relu')(input_img)\nencoded = Dense(64, activation='relu')(encoded)\nencoded = Dense(32,activation='relu')(encoded)\n\ndecoded = Dense(64,activation='relu')(encoded)\ndecoded = Dense(128,activation='relu')(decoded)\ndecoded = Dense(784,activation='sigmoid')(decoded)\n\nautoencoder = Model(input=input_img, output=decoded)\n\n\nencoder = Model(input=input_img, output = encoded)\nencoded_input = Input(shape=(32,))\n\ndecoder_layer1 = autoencoder.layers[-3]\ndecoder_layer2 = autoencoder.layers[-2]\ndecoder_layer3 = autoencoder.layers[-1]\n\ndecoder = Model(input=encoded_input, output= decoder_layer3(decoder_layer2(decoder_layer1(encoded_input))))\n\nautoencoder.compile(optimizer='adadelta',loss='binary_crossentropy')\n\nautoencoder.fit(x_train, x_train, nb_epoch=100, verbose=2,\n               batch_size=256,shuffle=True,\n               validation_data=(x_test, x_test))\n\nencoded_imgs = encoder.predict(x_test)\ndecoded_imgs = decoder.predict(encoded_imgs)\n\nplot_imgs(decoded_imgs)")

/home/felipe/venv2/local/lib/python2.7/site-packages/IPython/core/interactiveshell.pyc in run_cell_magic(self, magic_name, line, cell)
   2113             magic_arg_s = self.var_expand(line, stack_depth)
   2114             with self.builtin_trap:
-> 2115                 result = fn(magic_arg_s, cell)
   2116             return result
   2117 

<decorator-gen-59> in time(self, line, cell, local_ns)

/home/felipe/venv2/local/lib/python2.7/site-packages/IPython/core/magic.pyc in <lambda>(f, *a, **k)
    186     # but it's overkill for just that one bit of state.
    187     def magic_deco(arg):
--> 188         call = lambda f, *a, **k: f(*a, **k)
    189 
    190         if callable(arg):

/home/felipe/venv2/local/lib/python2.7/site-packages/IPython/core/magics/execution.pyc in time(self, line, cell, local_ns)
   1178         else:
   1179             st = clock2()
-> 1180             exec(code, glob, local_ns)
   1181             end = clock2()
   1182             out = None

<timed exec> in <module>()

/home/felipe/venv2/local/lib/python2.7/site-packages/keras/engine/training.pyc in fit(self, x, y, batch_size, nb_epoch, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight)
   1102                               verbose=verbose, callbacks=callbacks,
   1103                               val_f=val_f, val_ins=val_ins, shuffle=shuffle,
-> 1104                               callback_metrics=callback_metrics)
   1105 
   1106     def evaluate(self, x, y, batch_size=32, verbose=1, sample_weight=None):

/home/felipe/venv2/local/lib/python2.7/site-packages/keras/engine/training.pyc in _fit_loop(self, f, ins, out_labels, batch_size, nb_epoch, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics)
    820                 batch_logs['size'] = len(batch_ids)
    821                 callbacks.on_batch_begin(batch_index, batch_logs)
--> 822                 outs = f(ins_batch)
    823                 if type(outs) != list:
    824                     outs = [outs]

/home/felipe/venv2/local/lib/python2.7/site-packages/keras/backend/theano_backend.pyc in __call__(self, inputs)
    670     def __call__(self, inputs):
    671         assert type(inputs) in {list, tuple}
--> 672         return self.function(*inputs)
    673 
    674 

/home/felipe/venv2/local/lib/python2.7/site-packages/theano/compile/function_module.pyc in __call__(self, *args, **kwargs)
    857         t0_fn = time.time()
    858         try:
--> 859             outputs = self.fn()
    860         except Exception:
    861             if hasattr(self.fn, 'position_of_error'):

KeyboardInterrupt: 

In [ ]:

    

encoded = None
decoded = None

input_img = Input(shape=(1,28,28))
x = Convolution2D(16,3,3, activation='relu', border_mode='same')(input_img)
x = MaxPooling2D((2,2), border_mode='same')(x)
x = Convolution2D(8,3,3, activation='relu', border_mode='same')(x)
x = MaxPooling2D((2,2), border_mode='same')(x)
x = Convolution2D(8,3,3, activation='relu', border_mode='same')(x)

encoded = MaxPooling2D((2,2),border_mode='same')(x)

# at this point the representation has 8 x 4 x 4 = 128 dimensions
x = Convolution2D(8,3,3,activation='relu',border_mode='same')(encoded)
x = UpSampling2D((2,2))(x)
x = Convolution2D(8,3,3, activation='relu', border_mode='same')(x)
x = UpSampling2D((2,2))(x)
x = Convolution2D(16,3,3, activation='relu')(x)
x = UpSampling2D((2,2))(x)

decoded = Convolution2D(1,3,3, activation='sigmoid', border_mode='same')(x)

autoencoder = Model(input_img,decoded)
autoencoder.compile(optimizer='adadelta',loss='binary_crossentropy')

(x_train, _), (x_test, _) = mnist.load_data()

x_train = x_train.astype('float32') / 255.
x_test = x_test.astype('float32') / 255.
x_train = np.reshape(x_train, (len(x_train), 1, 28, 28))
x_test = np.reshape(x_test, (len(x_test), 1, 28, 28))

# the tutorial suggests we use tensorflow but I'll use theano

autoencoder.fit(x_train,x_train,
            verbose=2,
            nb_epoch=50,
            batch_size=128,
            shuffle=True,
            validation_data=(x_test,x_test))


    

In [ ]:

    

decoded_imgs = autoencoder.predict(x_test)


    

In [ ]:

    

plot_imgs(decoded_imgs)


    

In [ ]:

    

(x_train, _), (x_test, _) = mnist.load_data()

x_train = x_train.astype('float32') / 255.
x_test = x_test.astype('float32') / 255.
x_train = np.reshape(x_train, (len(x_train), 1, 28, 28))
x_test = np.reshape(x_test, (len(x_test), 1, 28, 28))

noise_factor= 0.5

x_train_noisy = x_train + noise_factor * np.random.normal(loc=0.0,scale=1.0,size=x_train.shape)
x_test_noisy = x_test + noise_factor * np.random.normal(loc=0.0,scale=1.0,size=x_test.shape)


    

In [ ]:

    

x_train_noisy = np.clip(x_train_noisy,0.,1.)
x_test_noisy = np.clip(x_test_noisy, 0.,1.)


    

In [ ]:

    

plot_imgs(x_test_noisy)


    

In [ ]:

    

input_img = Input(shape=(1,28,28))

x = Convolution2D(32,3,3,activation='relu',border_mode='same')(input_img)
x = MaxPooling2D((2,2), border_mode='same')(x)
x = Convolution2D(32,3,3,activation='relu',border_mode='same')(x)
encoded = MaxPooling2D((2,2),border_mode='same')(x)

# at this point the representation is (32,7,7)
x = Convolution2D(32,3,3,activation='relu',border_mode='same')(encoded)
x = UpSampling2D((2,2))(x)
x = Convolution2D(32,3,3,activation='relu',border_mode='same')(x)
x = UpSampling2D((2,2))(x)

decoded = Convolution2D(1,3,3,activation='sigmoid',border_mode='same')(x)

autoencoder = Model(input_img,decoded)

# decoder_layer1 = autoencoder.layers[-4]
# decoder_layer2 = autoencoder.layers[-3]
# decoder_layer3 = autoencoder.layers[-2]
# decoder_layer4 = autoencoder.layers[-1]

# encoded_input = Input(shape=(32,7,7))

# decoder = Model(
#     input=encoded_input, 
#     output= decoder_layer4(decoder_layer3(decoder_layer2(decoder_layer1(encoded_input)))))


autoencoder.compile(optimizer='adadelta',loss='binary_crossentropy')

# the source is noisy, but the target isn't?
autoencoder.fit(x_train_noisy,x_train,
               nb_epoch=100,
               batch_size=128,
               shuffle=True,
               validation_data=(x_test_noisy,x_test))


decoded_imgs = autoencoder.predict(x_test)
plot_imgs(decoded_imgs)


    

In [ ]: