In [1]:
from keras.layers import Input, Dense, Dropout, BatchNormalization
from keras.models import Model
from keras.datasets import mnist
from keras.callbacks import EarlyStopping
from keras.models import Sequential, load_model
from keras.optimizers import RMSprop
from keras.callbacks import TensorBoard
from __future__ import print_function
from IPython.display import SVG, Image
from keras import regularizers, Model
from matplotlib import rc
from keras.utils import plot_model
import keras
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import pydot
import graphviz
In [2]:
%matplotlib inline
font = {'family' : 'monospace',
'weight' : 'bold',
'size' : 20}
rc('font', **font)
In [3]:
num_classes = 10
input_dim = 784
batch_size = 256
(x_train, y_train), (x_val, y_val) = mnist.load_data()
x_train = x_train.astype('float32') / 255.
x_val = x_val.astype('float32') / 255.
# x_train = np.concatenate((x_train, x_val))
x_train = x_train.reshape((len(x_train), np.prod(x_train.shape[1:])))
x_val = x_val.reshape((len(x_val), np.prod(x_val.shape[1:])))
# print(x_train.shape)
y_train = keras.utils.to_categorical(y_train, num_classes)
y_val = keras.utils.to_categorical(y_val, num_classes)
In [4]:
hidden1_dim = 512
hidden2_dim = 512
In [5]:
input_data = Input(shape=(input_dim,), dtype='float32', name='main_input')
x = Dense(hidden1_dim, activation='relu', kernel_initializer='normal')(input_data)
x = Dropout(0.2)(x)
x = Dense(hidden2_dim, activation='relu', kernel_initializer='normal')(x)
x = Dropout(0.2)(x)
output_layer = Dense(num_classes, activation='sigmoid', kernel_initializer='normal')(x)
model = Model(input_data, output_layer)
model.compile(loss='binary_crossentropy',
optimizer=RMSprop(),
metrics=['accuracy'])
In [6]:
# model = keras.models.load_model('models/model.h5')
model.fit(x_train, y_train,
batch_size=batch_size,
epochs=20,
shuffle=True,
verbose=0,
validation_data=(x_val, y_val))
# model.save('models/model.h5')
Out[6]:
In [7]:
score = model.evaluate(x_val, y_val, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
In [8]:
fig = plt.figure(figsize=(20,10))
plt.plot(model.history.history['val_acc'])
plt.plot(model.history.history['acc'])
plt.axhline(y=score[1], c="red")
plt.text(0, score[1], "test: " + str(round(score[1], 4)), fontdict=font)
plt.title('model accuracy for neural net with 2 hidden layers')
plt.ylabel('accuracy')
plt.xlabel('epochs')
plt.legend(['valid', 'train'], loc='lower right')
plt.show()
In [9]:
plot_model(model, to_file='images/MNIST_FF_standard.png', show_shapes=True, show_layer_names=False, rankdir='TB')
In [10]:
Image("images/MNIST_FF_standard.png")
Out[10]:
In [11]:
encoding_dim1 = 128
encoding_dim2 = 64
encoding_dim3 = 32
decoding_dim1 = 64
decoding_dim2 = 128
decoding_dim3 = input_dim
epochs = 100
batch_size = 256
In [12]:
input_img = Input(shape=(input_dim,))
encoded = Dense(encoding_dim1, activation='relu')(input_img)
encoded = Dense(encoding_dim2, activation='relu')(encoded)
encoded = Dense(encoding_dim3, activation='relu')(encoded)
decoded = Dense(decoding_dim1, activation='relu')(encoded)
decoded = Dense(decoding_dim2, activation='relu')(decoded)
decoded = Dense(decoding_dim3, activation='sigmoid')(decoded)
In [13]:
deep_autoencoder = Model(input_img, decoded)
deep_autoencoder.compile(optimizer=RMSprop(), loss='binary_crossentropy')
In [14]:
# deep_autoencoder = keras.models.load_model('models/deep_autoencoder.h5')
deep_autoencoder.fit(x_train, x_train,
epochs=epochs,
batch_size=batch_size,
shuffle=True,
validation_data=(x_val, x_val))
# deep_autoencoder.save('models/deep_autoencoder.h5')
Out[14]:
In [15]:
score = deep_autoencoder.evaluate(x_val, x_val, verbose=0)
print(score)
In [16]:
decoded_imgs = deep_autoencoder.predict(x_val)
In [17]:
n = 10 # how many digits we will display
plt.figure(figsize=(20, 4))
for i in range(n):
# display original
ax = plt.subplot(2, n, i + 1)
plt.imshow(x_val[i].reshape(28, 28))
plt.gray()
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
# display reconstruction
ax = plt.subplot(2, n, i + 1 + n)
plt.imshow(decoded_imgs[i].reshape(28, 28))
plt.gray()
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
plt.show()
In [18]:
dc_encoded = Dense(encoding_dim1, activation='relu')(input_img)
dc_encoded = Dense(encoding_dim2, activation='relu')(dc_encoded)
dc_encoded = Dense(encoding_dim3, activation='relu')(dc_encoded)
dc_class_layer = Dense(num_classes, activation='sigmoid')(dc_encoded)
dc = Model(inputs=input_img, outputs=dc_class_layer)
dc.layers[1].set_weights(deep_autoencoder.layers[1].get_weights())
dc.layers[2].set_weights(deep_autoencoder.layers[2].get_weights())
dc.layers[3].set_weights(deep_autoencoder.layers[3].get_weights())
dc.compile(loss='binary_crossentropy', optimizer=RMSprop(), metrics=['accuracy'])
In [19]:
# dc = keras.models.load_model('models/dc.h5')
dc.fit(x_train, y_train
, epochs=7
, verbose=True
, batch_size=batch_size
, validation_data=(x_val, y_val)
, shuffle=True)
# dc.save('models/dc.h5')
Out[19]:
In [20]:
df_score = dc.evaluate(x_val, y_val)
print('Test loss:', df_score[0])
print('Test accuracy:', df_score[1])
In [21]:
encoding_dim1 = 128
input_img = Input(shape=(input_dim,))
encoded1 = Dense(encoding_dim1, activation='relu')(input_img)
decoded1 = Dense(input_dim, activation='relu')(encoded1)
class1 = Dense(num_classes, activation='softmax')(decoded1)
autoencoder1 = Model(input_img, class1)
autoencoder1.compile(optimizer=RMSprop(), loss='binary_crossentropy', metrics=['accuracy'])
encoder1 = Model(input_img, encoded1)
encoder1.compile(optimizer=RMSprop(), loss='binary_crossentropy')
In [22]:
plot_model(autoencoder1, to_file='images/sae_first_layer.png', show_shapes=True, show_layer_names=False, rankdir='TB')
In [23]:
Image('images/sae_first_layer.png')
Out[23]:
In [24]:
# autoencoder1 = keras.models.load_model('models/autoencoder1.h5')
# encoder1 = keras.models.load_model('models/encoder1.h5')
autoencoder1.fit(x_train
, y_train
, epochs=8
, batch_size=batch_size
, shuffle=True
, verbose=False
)
# autoencoder1.save('models/autoencoder1.h5')
# encoder1.save('models/encoder1.h5')
Out[24]:
In [25]:
score1 = autoencoder1.evaluate(x_val, y_val, verbose=0)
print('Test loss:', score1[0])
print('Test accuracy:', score1[1])
In [26]:
first_layer_code = encoder1.predict(x_train)
encoding_dim2 = 64
epoch2 = 5
encoded_2_input = Input(shape=(encoding_dim1,))
encoded2 = Dense(encoding_dim2, activation='relu')(encoded_2_input)
decoded2 = Dense(encoding_dim1, activation='relu')(encoded2)
class2 = Dense(num_classes, activation='softmax')(decoded2)
autoencoder2 = Model(encoded_2_input, class2)
autoencoder2.compile(optimizer=RMSprop(), loss='binary_crossentropy', metrics=['accuracy'])
encoder2 = Model(encoded_2_input, encoded2)
encoder2.compile(optimizer=RMSprop(), loss='binary_crossentropy')
In [27]:
plot_model(autoencoder2, to_file='images/sae_second_layer.png', show_shapes=True, show_layer_names=False, rankdir='TB')
In [28]:
Image('images/sae_second_layer.png')
Out[28]:
In [29]:
# autoencoder2 = keras.models.load_model('models/autoencoder2.h5')
# encoder2 = keras.models.load_model('models/encoder2.h5')
autoencoder2.fit(first_layer_code
, y_train
, epochs=epoch2
, batch_size=batch_size
, shuffle=True
, verbose=False
)
# autoencoder2.save('models/autoencoder2.h5')
# encoder2.save('models/encoder2.h5')
Out[29]:
In [30]:
first_layer_code_val = encoder1.predict(x_val)
score2 = autoencoder2.evaluate(first_layer_code_val, y_val, verbose=0)
print('Test loss:', score2[0])
print('Test accuracy:', score2[1])
In [31]:
second_layer_code = encoder2.predict(encoder1.predict(x_train))
encoding_dim3 = 32
epoch3 = 5
encoded_3_input = Input(shape=(encoding_dim2,))
encoded3 = Dense(encoding_dim3, activation='relu')(encoded_3_input)
decoded3 = Dense(encoding_dim2, activation='relu')(encoded3)
class3 = Dense(num_classes, activation='softmax')(decoded3)
autoencoder3 = Model(encoded_3_input, class3)
autoencoder3.compile(optimizer=RMSprop(), loss='binary_crossentropy', metrics=['accuracy'])
encoder3 = Model(encoded_3_input, encoded3)
encoder3.compile(optimizer=RMSprop(), loss='binary_crossentropy')
In [32]:
# autoencoder2 = keras.models.load_model('models/autoencoder2.h5')
# encoder2 = keras.models.load_model('models/encoder2.h5')
autoencoder3.fit(second_layer_code
, y_train
, epochs=epoch3
, batch_size=batch_size
, shuffle=True
, verbose=False
)
# autoencoder2.save('models/autoencoder2.h5')
# encoder2.save('models/encoder2.h5')
Out[32]:
In [33]:
second_layer_code_val = encoder2.predict(encoder1.predict(x_val))
score3 = autoencoder3.evaluate(second_layer_code_val, y_val, verbose=0)
print('Test loss:', score3[0])
print('Test accuracy:', score3[1])
In [34]:
sae_encoded1 = Dense(encoding_dim1, activation='relu')(input_img)
sae_encoded2 = Dense(encoding_dim2, activation='relu')(sae_encoded1)
sae_encoded3 = Dense(encoding_dim3, activation='relu')(sae_encoded2)
sae_decoded1 = Dense(encoding_dim2, activation='relu')(sae_encoded3)
sae_decoded2 = Dense(encoding_dim1, activation='relu')(sae_decoded1)
sae_decoded3 = Dense(input_dim, activation='sigmoid')(sae_decoded2)
sae = Model(input_img, sae_decoded3)
sae.layers[1].set_weights(autoencoder1.layers[1].get_weights())
sae.layers[2].set_weights(autoencoder2.layers[1].get_weights())
sae.layers[3].set_weights(autoencoder3.layers[1].get_weights())
# sae.layers[4].set_weights(autoencoder3.layers[2].get_weights())
# sae.layers[5].set_weights(autoencoder2.layers[2].get_weights())
# sae.layers[6].set_weights(autoencoder1.layers[2].get_weights())
sae.compile(loss='binary_crossentropy', optimizer=RMSprop())
In [35]:
sae.fit(x_train
, x_train
, epochs=10
, batch_size=batch_size
, shuffle=True
, verbose=False
)
Out[35]:
In [36]:
score4 = sae.evaluate(x_val, x_val, verbose=0)
print('Test loss:', score4)
In [37]:
decoded_imgs = sae.predict(x_val)
n = 10 # how many digits we will display
plt.figure(figsize=(20, 4))
for i in range(n):
# display original
ax = plt.subplot(2, n, i + 1)
plt.imshow(x_val[i].reshape(28, 28))
plt.gray()
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
# display reconstruction
ax = plt.subplot(2, n, i + 1 + n)
plt.imshow(decoded_imgs[i].reshape(28, 28))
plt.gray()
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
plt.show()
In [38]:
noise_factor = 0.5
x_train_noisy = x_train + noise_factor * np.random.normal(loc=0.0, scale=1.0, size=x_train.shape)
x_val_noisy = x_val + noise_factor * np.random.normal(loc=0.0, scale=1.0, size=x_val.shape)
# re-normalization by clipping to the intervall (0,1)
x_train_noisy = np.clip(x_train_noisy, 0., 1.)
x_val_noisy = np.clip(x_val_noisy, 0., 1.)
n = 10
plt.figure(figsize=(20, 2))
for i in range(n):
ax = plt.subplot(1, n, i + 1)
plt.imshow(x_val_noisy[i].reshape(28, 28))
plt.gray()
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
plt.show()
In [39]:
denoising_autoencoder = Model(input_img, sae_decoded3)
denoising_autoencoder.compile(optimizer=RMSprop(), loss='binary_crossentropy')
In [40]:
denoising_autoencoder.fit(x_train_noisy, x_train,
epochs=100,
batch_size=batch_size,
shuffle=True,
verbose=0)
Out[40]:
In [41]:
decoded_imgs = denoising_autoencoder.predict(x_val_noisy)
In [42]:
n = 10 # how many digits we will display
plt.figure(figsize=(20, 4))
for i in range(n):
# display original
ax = plt.subplot(2, n, i + 1)
plt.imshow(x_val[i].reshape(28, 28))
plt.gray()
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
# display reconstruction
ax = plt.subplot(2, n, i + 1 + n)
plt.imshow(decoded_imgs[i].reshape(28, 28))
plt.gray()
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
plt.show()
Classification of noisy data
In [43]:
score = model.evaluate(x_val_noisy, y_val, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
In [44]:
score = model.evaluate(decoded_imgs, y_val, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
In [45]:
input_img = Input(shape=(input_dim,))
sae_classifier_encoded1 = Dense(encoding_dim1, activation='relu')(input_img)
sae_classifier_encoded2 = Dense(encoding_dim2, activation='relu')(sae_classifier_encoded1)
sae_classifier_encoded3 = Dense(encoding_dim3, activation='relu')(sae_classifier_encoded2)
class_layer = Dense(num_classes, activation='softmax')(sae_classifier_encoded3)
sae_classifier = Model(inputs=input_img, outputs=class_layer)
sae_classifier.layers[1].set_weights(autoencoder1.layers[1].get_weights())
sae_classifier.layers[2].set_weights(autoencoder2.layers[1].get_weights())
sae_classifier.layers[3].set_weights(autoencoder3.layers[1].get_weights())
sae_classifier.compile(loss='binary_crossentropy', optimizer=RMSprop(), metrics=['accuracy'])
In [46]:
plot_model(sae_classifier, to_file='images/sae_classifier.png', show_shapes=True, show_layer_names=False, rankdir='TB')
In [47]:
Image('images/sae_classifier.png')
Out[47]:
In [48]:
# sae_classifier = keras.models.load_model('models/sae_classifier.h5')
sae_classifier.fit(x_train, y_train
, epochs=7
, verbose=False
, batch_size=batch_size
, shuffle=True)
# sae_classifier.save('models/sae_classifier.h5')
Out[48]:
In [49]:
score5 = sae_classifier.evaluate(x_val, y_val)
print('Test loss:', score5[0])
print('Test accuracy:', score5[1])
In [50]:
third_layer_code = encoder3.predict(encoder2.predict(encoder1.predict(x_train)))
encoding_dim4 = 2
encoded_4_input = Input(shape=(encoding_dim3,))
encoded4 = Dense(encoding_dim4, activation='sigmoid')(encoded_4_input)
decoded4 = Dense(encoding_dim3, activation='sigmoid')(encoded4)
class4 = Dense(num_classes, activation='softmax')(decoded4)
autoencoder4 = Model(encoded_4_input, class4)
autoencoder4.compile(optimizer=RMSprop(), loss='binary_crossentropy', metrics=['accuracy'])
encoder4 = Model(encoded_4_input, encoded4)
encoder4.compile(optimizer=RMSprop(), loss='binary_crossentropy')
In [51]:
autoencoder4.fit(third_layer_code
, y_train
, epochs=50
, batch_size=batch_size
, shuffle=True
, verbose=False
)
Out[51]:
In [53]:
third_layer_code_val = encoder3.predict(encoder2.predict(encoder1.predict(x_val)))
score4 = autoencoder4.evaluate(third_layer_code_val, y_val, verbose=0)
print('Test loss:', score4[0])
print('Test accuracy:', score4[1])
In [160]:
fourth_layer_code = encoder4.predict(encoder3.predict(encoder2.predict(encoder1.predict(x_train))))
In [161]:
value1 = [x[0] for x in fourth_layer_code]
value2 = [x[1] for x in fourth_layer_code]
y_classes = mnist.load_data()[0][1]
In [162]:
data = {'value1': value1, 'value2': value2, 'class' : y_classes}
data = pd.DataFrame.from_dict(data)
In [163]:
data.head()
Out[163]:
In [172]:
groups = data.groupby('class')
# Plot
fig, ax = plt.subplots(figsize=(20,10))
# plt.figure(figsize=(20,10))
ax.margins(0.05) # Optional, just adds 5% padding to the autoscaling
for name, group in groups:
ax.plot(group.value1, group.value2, marker='o', linestyle='', ms=3, label=name, alpha=0.7)
ax.legend()
plt.show()
In [173]:
third_layer_code = x_train
encoding_dim4 = 2
encoded_4_input = Input(shape=(input_dim,))
encoded4 = Dense(encoding_dim4, activation='sigmoid')(encoded_4_input)
decoded4 = Dense(input_dim, activation='sigmoid')(encoded4)
class4 = Dense(num_classes, activation='softmax')(decoded4)
autoencoder4 = Model(encoded_4_input, class4)
autoencoder4.compile(optimizer=RMSprop(), loss='binary_crossentropy', metrics=['accuracy'])
encoder4 = Model(encoded_4_input, encoded4)
encoder4.compile(optimizer=RMSprop(), loss='binary_crossentropy')
In [174]:
autoencoder4.fit(third_layer_code
, y_train
, epochs=50
, batch_size=batch_size
, shuffle=True
, verbose=False
)
Out[174]:
In [175]:
third_layer_code_val = x_val
score4 = autoencoder4.evaluate(third_layer_code_val, y_val, verbose=0)
print('Test loss:', score3[0])
print('Test accuracy:', score3[1])
In [176]:
fourth_layer_code = encoder4.predict(x_train)
In [177]:
value1 = [x[0] for x in fourth_layer_code]
value2 = [x[1] for x in fourth_layer_code]
y_classes = mnist.load_data()[0][1]
In [178]:
data = {'value1': value1, 'value2': value2, 'class' : y_classes}
data = pd.DataFrame.from_dict(data)
In [163]:
data.head()
Out[163]:
In [179]:
groups = data.groupby('class')
# Plot
fig, ax = plt.subplots(figsize=(20,10))
# plt.figure(figsize=(20,10))
ax.margins(0.05) # Optional, just adds 5% padding to the autoscaling
for name, group in groups:
ax.plot(group.value1, group.value2, marker='o', linestyle='', ms=3, label=name, alpha=0.7)
ax.legend()
plt.show()