This notebook is a modified fork of the Bayesian MNIST classifier implementation here.
In this notebook, a Bayesian LeNet model is trained using the MNIST data.
A Bayesian inference function generates the mean prediction accuracy and the associated prediction uncertainty of the trained model.
In [1]:
! wget https://media.githubusercontent.com/media/rahulremanan/python_tutorial/master/Machine_Vision/07_Bayesian_deep_learning/weights/bayesianLeNet.h5 -O ./bayesianLeNet.h5
In [2]:
from keras import Input, Model
from keras.layers import Conv2D, MaxPool2D, Dense, Flatten, Dropout
In [0]:
def LeNet(input_shape, num_classes):
inp = Input(shape=input_shape)
x = Conv2D(filters=20, kernel_size=5, strides=1)(inp)
x = MaxPool2D(pool_size=2, strides=2)(x)
x = Conv2D(filters=50, kernel_size=5, strides=1)(x)
x = MaxPool2D(pool_size=2, strides=2)(x)
x = Flatten()(x)
x = Dense(500, activation='relu')(x)
x = Dense(num_classes, activation='softmax')(x)
return Model(inp, x, name='LeNet')
In [0]:
def bayesianLeNet(input_shape, num_classes, enable_dropout=True):
"""
An example implementation of a Bayesian LeNet convolutional neural network.
This network uses the Bayesian approximation by Monte Carlo estimations using dropouts.
To enable Bayesian approxiamtion, set the enable_dropout flag to True.
"""
inp = Input(shape=input_shape)
x = Conv2D(filters=20, kernel_size=5, strides=1)(inp)
x = Dropout(0.5)(x, training=True)
x = MaxPool2D(pool_size=2, strides=2)(x)
x = Conv2D(filters=50, kernel_size=5, strides=1)(x)
x = Dropout(0.5)(x, training=enable_dropout)
x = MaxPool2D(pool_size=2, strides=2)(x)
x = Flatten()(x)
x = Dropout(0.5)(x, training=enable_dropout)
x = Dense(500, activation='relu')(x)
x = Dropout(0.5)(x, training=enable_dropout)
x = Dense(num_classes, activation='softmax')(x)
return Model(inp, x, name='bayesianLeNet')
In [0]:
import argparse
import os
from keras.callbacks import TensorBoard
from keras.datasets import mnist
from keras import utils
import numpy as np
from tqdm import tqdm
In [0]:
TENSORBOARD_DIR = './tensorboard'
MODEL_PATH = './bayesianLeNet.h5'
In [0]:
def make_dirs():
if not os.path.isdir(TENSORBOARD_DIR):
os.makedirs(TENSORBOARD_DIR)
In [0]:
make_dirs()
In [0]:
def prepare_data():
(X_train, y_train), (X_test, y_test) = mnist.load_data()
X_train = X_train.reshape((X_train.shape[0], X_train.shape[1], X_train.shape[2], 1))
X_train = X_train.astype(np.float32) / 255.
X_test = X_test.reshape((X_test.shape[0], X_test.shape[1], X_test.shape[2], 1))
X_test = X_test.astype(np.float32) / 255.
y_train, y_test = utils.to_categorical(y_train, 10), utils.to_categorical(y_test, 10)
return (X_train, y_train), (X_test, y_test)
In [10]:
(X_train, y_train), (X_test, y_test) = prepare_data()
In [0]:
bayesian_network=True
download_weights=True
batch_size=1000
epochs=10
In [12]:
if bayesian_network:
model = bayesianLeNet(input_shape=X_train.shape[1:],
num_classes=10)
else:
model = LeNet(input_shape=X_train.shape[1:],
num_classes=10)
In [0]:
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['acc'])
In [14]:
if os.path.exists(MODEL_PATH):
model.load_weights(MODEL_PATH)
print ('Loaded model weights from: {}'.format(MODEL_PATH))
In [15]:
model.fit(x=X_train,
y=y_train,
batch_size=batch_size,
epochs=epochs,
validation_data=(X_test,
y_test),
callbacks=[TensorBoard(log_dir=os.path.join(TENSORBOARD_DIR,
model.name),
write_images=True)])
Out[15]:
In [0]:
model.save_weights(MODEL_PATH)
In [0]:
def bayesianInference(model, X_test, y_test, eval_steps=10):
batch_size = 1000
bayesian_error = []
for batch_id in tqdm(range(X_test.shape[0] // batch_size)):
# take batch of data
x = X_test[batch_id * batch_size: (batch_id + 1) * batch_size]
# init empty predictions
y_ = np.zeros((eval_steps, batch_size, y_test[0].shape[0]))
for sample_id in range(eval_steps):
# save predictions from a sample pass
y_[sample_id] = model.predict(x, batch_size)
# average over all passes
mean_y = y_.mean(axis=0)
# evaluate against labels
y = y_test[batch_size * batch_id: (batch_id + 1) * batch_size]
# compute error
point_error = np.count_nonzero(np.not_equal(mean_y.argmax(axis=1), y.argmax(axis=1)))
bayesian_error.append(point_error)
mean_error = np.sum(bayesian_error) / X_test.shape[0]
uncertainty = np.std(bayesian_error) / X_test.shape[0]
mean_accuracy = 1 - mean_error
return [mean_accuracy, uncertainty]
In [18]:
if bayesian_network:
out = bayesianInference(model, X_test, y_test)
print ('\n')
print ('\nValidation accuracy: {} ...'.format(out[0]))
print ('Validation uncertainty: {} ...'.format(out[1]))
else:
(_, acc) = model.evaluate(x=X_test,
y=y_test,
batch_size=args.batch_size)
print('\nValidation accuracy: {}'.format(acc))
In [0]:
if download_weights:
from google.colab import files
files.download(MODEL_PATH)