This is a demonstration notebook on: deep Bayesian networks using Monte Carlo dropouts. In this notebook, an Inception-ResNet version 2 base network is combined with a multi-layer Bayesian perceptron architecture. The Bayesian network was built using dropout, simulating tiny changes to the architecture, to compute the model uncertainty associated with each predictions.
Monte Carlo estimations using dropout, is a Bayesain approach to quantify the model uncertainty. By quantifying uncertainty, harder questions such as trust worthiness of a decision made by a neural network, can later be validated against out-of-sample, real world examples.
The idea of using dropout layers as a scalable, automated mechanism to construct a deep Bayesian network, was first put forth by Yarin Gal; in his PhD thesis paper at Cambridge University.
Read the original work Dropout as a Bayesian Approximation by Yarin Gal, University of Cambridge (Via arxiv)
In [0]:
setup=True
download_data=True
enable_bayesian_optimizer=False
enable_random_optimizer=False
enable_training=True
load_weights=True
upload_weights = True
dataset_id='dogs_vs_cats'
val_fn='val_f1'
loss_fn='categorical_crossentropy'
train_aug=True
val_aug=False
gen_predictions=True
detection_threshold=0.25
colab_mode = True
fetch_raw_data = False
upload_data = False
fine_tune = True
verbose = False
enable_dropout = True
bayesian_cam = True
MODEL_NAME='InceptionResNetV2_{}.model'.format(dataset_id)
EPOCHS=1
NB_FROZEN_LAYERS = 45
LEARNING_RATE = 9.31579590417492e-05
INPUT_SHAPE = (299,299,3)
BATCH_SIZE = 2
STEPS_PER_EPOCHS = 400
FC_SIZE = 65
WEIGHTS_DECAY = 0.01
DROPOUT = 0.5
ACTIVATION='tanh'
OPTIMIZER='adam'
OPTIMIZERS=['adam_amsgrad','adadelta','sgd','adam']
ACTIVATIONS=['relu','tanh','sigmoid','elu']
LEARNING_RATES=[0.1, 0.01, 1e-3]
path_to_train = './{}/train/'.format(dataset_id)
train_labels = './train_{}.csv'.format(dataset_id)
checkpointer_savepath = './{}/{}'.format(dataset_id, MODEL_NAME)
In [0]:
import os
import sys
import subprocess
import gc
In [0]:
def execute_in_shell(command=None,
verbose = False):
"""
command -- keyword argument, takes a list as input
verbsoe -- keyword argument, takes a boolean value as input
This is a function that executes shell scripts from within python.
Keyword argument 'command', should be a list of shell commands.
Keyword argument 'verbose', should be a boolean value to set verbose level.
Example usage: execute_in_shell(command = ['ls ./some/folder/',
ls ./some/folder/ -1 | wc -l'],
verbose = True )
This command returns dictionary with elements: Output and Error.
Output records the console output,
Error records the console error messages.
"""
error = []
output = []
if isinstance(command, list):
for i in range(len(command)):
try:
process = subprocess.Popen(command[i], shell=True, stdout=subprocess.PIPE)
process.wait()
out, err = process.communicate()
error.append(err)
output.append(out)
if verbose:
print ('Success running shell command: {}'.format(command[i]))
except Exception as e:
print ('Failed running shell command: {}'.format(command[i]))
if verbose:
print(type(e))
print(e.args)
print(e)
else:
print ('The argument command takes a list input ...')
return {'Output': output, 'Error': error }
In [0]:
command = ['pip3 install -q kaggle PyDrive scikit-optimize >/dev/null 2>&1',
'mkdir /content/',
'mkdir /content/.kaggle/',
'mkdir ./{}/'.format(dataset_id)]
In [0]:
if setup and colab_mode:
execute_in_shell(command = command,
verbose = True)
In [0]:
if colab_mode:
from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
from google.colab import auth
from oauth2client.client import GoogleCredentials
from googleapiclient.http import MediaIoBaseDownload
import io
import glob
import fnmatch
import random
from multiprocessing import Process
import os, sys, math
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from PIL import Image
import cv2
from imgaug import augmenters as iaa
from tqdm import tqdm
import warnings
warnings.filterwarnings("ignore")
In [0]:
import argparse
import os
import random
import time
import sys
import glob
try:
import h5py
except:
print ('Package h5py needed for saving model weights ...')
sys.exit(1)
import json
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
try:
import tensorflow
import keras
except:
print ('This code uses tensorflow deep-learning framework and keras api ...')
print ('Install tensorflow and keras to train the classifier ...')
sys.exit(1)
import PIL
from collections import defaultdict
from keras.applications.inception_v3 import InceptionV3, \
preprocess_input as preprocess_input_inceptionv3
from keras.applications.inception_resnet_v2 import InceptionResNetV2, \
preprocess_input as preprocess_input_inceptionv4
from keras.models import Model, \
model_from_json, \
load_model
from keras.layers import Dense, \
GlobalAveragePooling2D, \
Dropout, \
BatchNormalization
from keras.layers.merge import concatenate
from keras.preprocessing.image import ImageDataGenerator
from keras.regularizers import l2
from keras.optimizers import SGD, \
RMSprop, \
Adagrad, \
Adadelta, \
Adam, \
Adamax, \
Nadam
from keras.callbacks import EarlyStopping, \
ModelCheckpoint, \
ReduceLROnPlateau
from multiprocessing import Process
In [0]:
if setup and fetch_raw_data and colab_mode:
from google.colab import files
uploaded = files.upload()
In [0]:
command = ['mv ./*.json /content/.kaggle/',
'cp /content/.kaggle/kaggle.json ~/.kaggle/kaggle.json',
'cp /content/.kaggle/kaggle.json /root/.kaggle/kaggle.json',
'chmod 600 ~/.kaggle/kaggle.json',
'kaggle competitions download -c dogs-vs-cats-redux-kernels-edition',
'mv /content/*.csv ./',
'mv /content/*.zip ./',
'mv ./train.zip ./train_{}.zip'.format(dataset_id),
'mv ./test.zip ./test_{}.zip'.format(dataset_id),
'mv ./sample_submission.csv ./sample_submission_{}'.format(dataset_id)]
In [0]:
if fetch_raw_data:
execute_in_shell(command = command, verbose = True)
In [0]:
def cloud_authenticate():
auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
drive = GoogleDrive(gauth)
print ("Sucessfully authenticated to access Google Drive ...")
return drive
In [0]:
if colab_mode:
drive = cloud_authenticate()
In [0]:
def googledrive_fetch(file_name = None,
fetch=True,
fetch_by_id = False,
latest = True,
file_id = None,
multi_file = False):
"""
A function that fetches files from Google Drive.
The function takes five keyword arguments:
file_name -- Passes the file name string
fetch -- Specify if a file name should be downloaded
fetch_by_id -- Specify a file to be downloaded by file id
multi_file -- Download all the files with the same file name from Google Drive
"""
query = 'title='+"'"+file_name+"'"
try:
file_list=drive.ListFile({'q': "{}".format(query)}).GetList()
except:
return ("Error finding file with {}".format(query))
if len(file_list) >1:
print ("A total of {} files with the same file name found ...".format(len(file_list)))
for f in file_list:
title = f['title']
id = f.metadata.get('id')
print ("Found: {} file, with file id: {}".format(title, id))
if multi_file:
print ("Downloading {} files with file name {}".format(len(file_list), title))
print ("Staring download ...")
elif latest:
print ("Downloading the most recent {} file ...".format(title))
elif file_id == None:
print ("Set keyword argument fetch_by_id = True and specify id using keyword argument file_id = 'id' to download a specific file ...")
print ("--OR--")
print ("Set keyword argument multi_file = True to automatically download all the files ...")
return None
else:
print ("Starting download ...")
n = 0
if latest:
try:
title = file_list[0]['title']
except:
return ("Error finding file with {}".format(query))
latest_file_id = file_list[0].metadata.get('id')
print ("Found most recent version of: {} file with file id: {} ...".format(title, latest_file_id))
for f in file_list:
if fetch and multi_file and n>0:
save_path = os.path.join('./'+str(n)+'_'+file_name)
else:
save_path = os.path.join('./'+file_name)
title = f['title']
if fetch_by_id and file_id !=None:
id = file_id
elif latest:
id = latest_file_id
elif fetch_by_id and file_id == None:
print ('Please specify the file id for downloading using the file_id argument ...')
else:
id = f.metadata.get('id')
print ("Downloading {} file, with file id: {} ...".format(title, id))
if fetch or fetch_by_id or latest:
local_file = io.FileIO(save_path, mode='wb')
try:
request = drive.auth.service.files().get_media(fileId=id)
downloader = MediaIoBaseDownload(local_file, request, chunksize=2048*102400)
done = False
while done is False:
status, done = downloader.next_chunk()
except:
return 'Downloading failed ...'
local_file.close()
print ("Successfully downloaded the file: {} to: {} ...".format(file_name, save_path))
if fetch_by_id and file_id !=None:
return None
elif latest:
return None
elif n >= 0:
print ("Downloaded {} of {} files ...".format(n+1, len(file_list)))
else:
print ("Download failed ...")
n +=1
return None
In [0]:
def googledrive_save(file_name = None,
file_dir = None,
upload = False,
prefix = None):
if upload == True and file_name != None and file_dir !=None:
try:
if prefix != None:
file = drive.CreateFile({'title': str(prefix) + str(file_name) })
else:
file = drive.CreateFile({'title': str(file_name) })
file.SetContentFile(os.path.join(file_dir + str(file_name)))
file.Upload()
print (str(file_name) + " successfully uploaded to Google drive ...")
except:
print ("Failed to save :" + str(file_name) + " to Google drive ...")
In [0]:
file_dir = './'
file_name = ['train_{}.zip'.format(dataset_id),
'train_{}.csv'.format(dataset_id),
'test_{}.zip'.format(dataset_id),
'sample_submission_{}.csv'.format(dataset_id),
'Transfer_learn_299_299_{}.h5'.format(dataset_id),
'{}'.format(MODEL_NAME)]
In [0]:
if upload_data and colab_mode:
for f in file_name:
googledrive_save(file_name = f,
file_dir = file_dir,
upload = True)
In [0]:
if download_data and colab_mode:
for f in file_name:
googledrive_fetch(file_name = f,
fetch=True,
latest = True)
In [0]:
command = ['mkdir ./{}/'.format(dataset_id),
'mkdir ./{}/train/'.format(dataset_id),
'sudo apt-get install p7zip-full',
'7z e ./train_{}.zip -o./{}/train/ -r'.format(dataset_id,
dataset_id),
'rm ./train_{}.zip'.format(dataset_id)]
In [0]:
if setup:
execute_in_shell(command = command, verbose = True)
In [0]:
command = ['mkdir ./{}/'.format(dataset_id),
'mkdir ./{}/test/'.format(dataset_id),
'mkdir ./{}/train/'.format(dataset_id),
'mkdir ./{}/train/cats/'.format(dataset_id),
'mkdir ./{}/train/dogs/'.format(dataset_id),
'unzip -q ./train_{}.zip -d ./{}/train/'.format(dataset_id,
dataset_id),
'unzip -q ./test_{}.zip -d ./{}/test/'.format(dataset_id,
dataset_id),
'mv ./{}/train/train/cat*.jpg ./{}/train/cats/'.format(dataset_id,
dataset_id),
'mv ./{}/train/train/dog*.jpg ./{}/train/dogs/'.format(dataset_id,
dataset_id),
"find ./{}/train/ -name 'dog*.jpg' -exec mv --target-directory=./{}/train/dogs/ '{}' +".format(dataset_id,
dataset_id,
{}),
"find ./{}/train/ -name 'cat*.jpg' -exec mv --target-directory=./{}/train/cats/ '{}' +".format(dataset_id,
dataset_id,
{}),
'mv ./{}/train/dog*.jpg ./{}/train/dogs/'.format(dataset_id,
dataset_id),
'mv ./{} ./{}/{}'.format(MODEL_NAME,
dataset_id,
MODEL_NAME),
'mkdir ./{}/checkpoint/'.format(dataset_id),
'mv ./Transfer_learn_299_299_{}.h5 ./{}/checkpoint/Transfer_learn_299_299_.h5'.format(dataset_id,
dataset_id),
'rm ./*.zip',
'rm -r ./{}/train/train'.format(dataset_id)]
In [0]:
if setup:
execute_in_shell(command = command, verbose = True)
In [0]:
def generate_timestamp():
"""
A function to generate time-stamp information.
Calling the function returns a string formatted current system time.
Eg: 2018_10_10_10_10_10
Example usage: generate_timestamp()
"""
timestring = time.strftime("%Y_%m_%d-%H_%M_%S")
print ("Time stamp generated: " + timestring)
return timestring
In [0]:
timestr = generate_timestamp()
In [0]:
def is_valid_file(parser, arg):
"""
A function that checks if a give file path contains a valid file or not.
The function returns the full file path if there is a valid file persent.
If there is no valid file present at a file path location, it returns a parser error message.
Takes two positional arguments: parser and arg
Example usage:
import argsparse
a = argparse.ArgumentParser()
a.add_argument("--file_path",
help = "Check if a file exists in the specified file path ...",
dest = "file_path",
required=False,
type=lambda x: is_valid_file(a, x),
nargs=1)
args = a.parse_args()
args = get_user_options()
"""
if not os.path.isfile(arg):
try:
parser.error("The file %s does not exist ..." % arg)
return None
except:
if parser != None:
print ("No valid argument parser found ...")
print ("The file %s does not exist ..." % arg)
return None
else:
print ("The file %s does not exist ..." % arg)
return None
else:
return arg
In [0]:
def is_valid_dir(parser, arg):
"""
This function checks if a directory exists or not.
It can be used inside the argument parser.
Example usage:
import argsparse
a = argparse.ArgumentParser()
a.add_argument("--dir_path",
help = "Check if a file exists in the specified file path ...",
dest = "file_path",
required=False,
type=lambda x: is_valid_dir(a, x),
nargs=1)
args = a.parse_args()
args = get_user_options()
"""
if not os.path.isdir(arg):
try:
return parser.error("The folder %s does not exist ..." % arg)
except:
if parser != None:
print ("No valid argument parser found")
print ("The folder %s does not exist ..." % arg)
return None
else:
print ("The folder %s does not exist ..." % arg)
return None
else:
return arg
In [0]:
def string_to_bool(val):
"""
A function that checks if an user argument is boolean or not.
Example usage:
import argsparse
a = argparse.ArgumentParser()
a.add_argument("--some_bool_arg",
help = "Specify a boolean argument ...",
dest = "some_bool_arg",
required=False,
default=[True],
nargs=1,
type = string_to_bool)
args = a.parse_args()
args = get_user_options()
"""
if val.lower() in ('yes', 'true', 't', 'y', '1', 'yeah', 'yup'):
return True
elif val.lower() in ('no', 'false', 'f', 'n', '0', 'none', 'nope'):
return False
else:
raise argparse.ArgumentTypeError('Boolean value expected ...')
In [0]:
def activation_val(val):
activation_function_options = ('hard_sigmoid',
'elu',
'linear',
'relu',
'selu',
'sigmoid',
'softmax',
'softplus',
'sofsign',
'tanh')
if val.lower() in activation_function_options:
return val
else:
raise argparse.ArgumentTypeError('Unexpected activation function. \
\nExpected values are: {} ...'.format(activation_function_options))
In [0]:
def loss_val(val):
loss_function_options = ('mean_squared_error',
'mean_absolute_error',
'mean_absolute_percentage_error',
'mean_squared_logarithmic_error',
'squared_hinge',
'hinge',
'categorical_hinge',
'logcosh',
'categorical_crossentropy',
'sparse_categorical_crossentropy',
'binary_crossentropy',
'kullback_leibler_divergence',
'poisson',
'cosine_proximity')
if val.lower() in loss_function_options:
return val
else:
raise argparse.ArgumentTypeError('Unexpected loss function. \
\nExpected values are: {} ...'.format(loss_function_options))
In [0]:
def get_nb_files(directory):
if not os.path.exists(directory):
return 0
cnt = 0
for r, dirs, files in os.walk(directory):
for dr in dirs:
cnt += len(glob.glob(os.path.join(r, dr + "/*")))
return cnt
In [0]:
def add_top_layer(args, enable_dropout, base_model, nb_classes):
"""
This functions adds a fully connected convolutional neural network layer to a base model.
The required input arguments for this function are: args, base_model and nb_classes.
args: argument inputs the user arguments to be passed to the function,
base_model: argument inputs the base model architecture to be added to the top layer,
nb_classes: argument inputs the total number of classes for the output layer.
"""
try:
dropout = float(args.dropout[0])
weight_decay = float(args.decay[0])
except:
dropout = DEFAULT_DROPOUT
print ('Invalid input for dropout ...')
try:
activation = str(args.activation[0]).lower()
print ('Building model using activation function: ' + str(activation))
except:
activation = 'relu'
print ('Invalid input for activation function ...')
print ('Choice of activation functions: hard_sigmoid, elu, linear, relu, selu, sigmoid, softmax, softplus, sofsign, tanh ...')
print ('Building model using default activation function: relu')
bm = base_model.output
x = Dropout(dropout,
name='dropout_fc1')(bm,
training=enable_dropout)
x = GlobalAveragePooling2D(name='gloablAveragePooling2D_fc1')(x)
x = Dropout(dropout,
name='dropout_fc2')(x,
training=enable_dropout)
x = BatchNormalization(name='batchNormalization_fc1')(x)
x = Dense(FC_SIZE,
activation=activation,
kernel_regularizer=l2(weight_decay),
name='dense_fc1')(x)
x = Dropout(dropout,
name='dropout_fc3')(x,
training=enable_dropout)
x1 = Dense(FC_SIZE,
activation=activation,
kernel_regularizer=l2(weight_decay),
name="dense_fc2")(x)
x1 = Dropout(dropout,
name = 'dropout_fc4')(x1,
training=enable_dropout)
x1 = BatchNormalization(name="batchNormalization_fc2")(x1)
x1 = Dense(FC_SIZE,
activation=activation,
kernel_regularizer=l2(weight_decay),
name="dense_fc3")(x1)
x1 = Dropout(dropout,
name = 'dropout_fc5')(x1,
training=enable_dropout)
x2 = Dense(FC_SIZE,
activation=activation,
kernel_regularizer=l2(weight_decay),
name="dense_fc4")(x)
x2 = Dropout(dropout,
name = 'dropout_fc6')(x2,
training=enable_dropout)
x2 = BatchNormalization(name="batchNormalization_fc3")(x2)
x2 = Dense(FC_SIZE,
activation=activation,
kernel_regularizer=l2(weight_decay),
name="dense_fc5")(x2)
x2 = Dropout(dropout,
name = 'dropout_fc7')(x2,
training=enable_dropout)
x12 = concatenate([x1, x2], name = 'mixed11')
x12 = Dropout(dropout,
name = 'dropout_fc8')(x12,
training=enable_dropout)
x12 = Dense(FC_SIZE//16,
activation=activation,
kernel_regularizer=l2(weight_decay),
name = 'dense_fc6')(x12)
x12 = Dropout(dropout,
name = 'dropout_fc9')(x12,
training=enable_dropout)
x12 = BatchNormalization(name="batchNormalization_fc4")(x12)
x12 = Dense(FC_SIZE//32,
activation=activation,
kernel_regularizer=l2(weight_decay),
name = 'dense_fc7')(x12)
x12 = Dropout(dropout,
name = 'dropout_fc10')(x12,
training=enable_dropout)
x3 = Dropout(dropout,
name='dropout_fc11')(bm,
training=enable_dropout)
x3 = GlobalAveragePooling2D( name = 'globalAveragePooling2D_fc2')(x3)
x3 = Dense(FC_SIZE//2,
activation=activation,
kernel_regularizer=l2(weight_decay),
name = 'dense_fc8')(x3)
x3 = Dropout(dropout,
name = 'dropout_fc12')(x3,
training=enable_dropout)
x3 = BatchNormalization(name="batchNormalization_fc5")(x3)
x3 = Dense(FC_SIZE//2,
activation=activation,
kernel_regularizer=l2(weight_decay),
name = 'dense_fc9')(x3)
x3 = Dropout(dropout,
name = 'dropout_fc13')(x3,
training=enable_dropout)
xout = concatenate([x12, x3], name ='mixed12')
xout = Dense(FC_SIZE//32,
activation= activation,
kernel_regularizer=l2(weight_decay),
name = 'dense_fc10')(xout)
xout = Dropout(dropout,
name = 'dropout_fc14')(xout,
training=enable_dropout)
predictions = Dense(nb_classes, \
activation='softmax', \
kernel_regularizer=l2(weight_decay),
name='prediction')(xout) # Softmax output layer
model = Model(inputs=base_model.input,
outputs=predictions)
return model
In [0]:
def finetune_model(model, optimizer, loss, NB_FROZEN_LAYERS):
"""
A function that freezes the bottom NB_LAYERS and retrain the remaining top layers.
The required input arguments for this function are: model, optimizer and NB_FROZEN_LAYERS.
model: inputs a model architecture with base layers to be frozen during training,
optimizer: inputs a choice of optimizer value for compiling the model,
loss: inputs a choice for loss function used for compiling the model,
NB_FROZEN_LAYERS: inputs a number that selects the total number of base layers to be frozen during training.
"""
for layer in model.layers[:NB_FROZEN_LAYERS]:
layer.trainable = False
for layer in model.layers[NB_FROZEN_LAYERS:]:
layer.trainable = True
model.compile(optimizer=optimizer,
loss=loss,
metrics=['accuracy'])
return model
In [0]:
def transferlearn_model(model, base_model, optimizer, loss):
"""
Function that freezes the base layers to train just the top layer.
This function takes three positional arguments:
model: specifies the input model,
base_model: specifies the base model architecture,
optimizer: optimizer function for training the model,
loss: loss function for compiling the model
Example usage:
transferlearn_model(model, base_model, optimizer)
"""
for layer in base_model.layers:
layer.trainable = False
model.compile(optimizer=optimizer,
loss=loss,
metrics=['accuracy'])
return model
In [0]:
def save_model(args, name, model):
file_loc = args.output_dir[0]
file_pointer = os.path.join(file_loc+"//trained_"+ timestr)
model.save_weights(os.path.join(file_pointer + "_weights"+str(name)+".model"))
model_json = model.to_json() # Serialize model to JSON
with open(os.path.join(file_pointer+"_config"+str(name)+".json"), "w") as json_file:
json_file.write(model_json)
print ("Saved the trained model weights to: " +
str(os.path.join(file_pointer + "_weights"+str(name)+".model")))
print ("Saved the trained model configuration as a json file to: " +
str(os.path.join(file_pointer+"_config"+str(name)+".json")))
In [0]:
def generate_labels(args):
file_loc = args.output_dir[0]
file_pointer = os.path.join(file_loc+"//trained_labels")
data_dir = args.train_dir[0]
val_dir_ = args.val_dir[0]
dt = defaultdict(list)
dv = defaultdict(list)
for root, subdirs, files in os.walk(data_dir):
for filename in files:
file_path = os.path.join(root, filename)
assert file_path.startswith(data_dir)
suffix = file_path[len(data_dir):]
suffix = suffix.lstrip("/")
label = suffix.split("/")[0]
dt[label].append(file_path)
for root, subdirs, files in os.walk(val_dir_):
for filename in files:
file_path = os.path.join(root, filename)
assert file_path.startswith(val_dir_)
suffix = file_path[len(val_dir_):]
suffix = suffix.lstrip("/")
label = suffix.split("/")[0]
dv[label].append(file_path)
labels = sorted(dt.keys())
val_labels = sorted(dv.keys())
if set(labels) == set (val_labels):
print("\nTraining labels: " + str(labels))
print("\nValidation labels: " + str(val_labels))
with open(os.path.join(file_pointer+".json"), "w") as json_file:
json.dump(labels, json_file)
else:
print("\nTraining labels: " + str(labels))
print("\nValidation labels: " + str(val_labels))
print ("Mismatched training and validation data labels ...")
print ("Sub-folder names do not match between training and validation directories ...")
sys.exit(1)
return labels
In [0]:
def normalize(args,
labels,
move = False,
sub_sample = False):
if args.normalize[0] and os.path.exists(args.root_dir[0]):
commands = ["rm -r {}/.tmp_train/".format(args.root_dir[0]),
"rm -r {}/.tmp_validation/".format(args.root_dir[0]),
"mkdir {}/.tmp_train/".format(args.root_dir[0]),
"mkdir {}/.tmp_validation/".format(args.root_dir[0])]
execute_in_shell(command=commands,
verbose=verbose)
del commands
mk_train_folder = "mkdir -p {}/.tmp_train/".format(args.root_dir[0]) + "{}"
mk_val_folder = "mkdir -p {}/.tmp_validation/".format(args.root_dir[0]) + "{}"
train_class_sizes = []
val_class_sizes = []
for label in labels:
train_class_sizes.append(len(glob.glob(args.train_dir[0] + "/{}/*".format(label))))
val_class_sizes.append(len(glob.glob(args.val_dir[0] + "/{}/*".format(label))))
train_size = min(train_class_sizes)
val_size = min(val_class_sizes)
if sub_sample and 0 <= args.train_sub_sample[0] <=1 and 0 <= args.val_sub_sample[0] <=1 :
train_size = int(train_size * args.train_sub_sample[0])
val_size = int(val_size * args.val_sub_sample[0])
print ("Normalized training class size {}".format(train_size))
print ("Normalized validation class size {}".format(val_size))
for label in labels:
commands = [mk_train_folder.format(label),
mk_val_folder.format(label)]
execute_in_shell(command=commands,
verbose=verbose)
del commands
commands = []
for label in labels:
train_images = (glob.glob('{}/{}/*.*'.format(args.train_dir[0], label), recursive=True))
val_images = (glob.glob('{}/{}/*.*'.format(args.val_dir[0], label), recursive=True))
sys_rnd = random.SystemRandom()
if move:
cmd = 'mv'
else:
cmd = 'cp'
for file in sys_rnd.sample(train_images, train_size):
if os.path.exists(file):
commands.append('{} {} ./.tmp_train/{}/'.format(cmd, file, label))
for file in sys_rnd.sample(val_images, val_size):
if os.path.exists(file):
commands.append('{} {} ./.tmp_validation/{}/'.format(cmd, file, label))
p = Process(target=execute_in_shell, args=([commands]))
p.start()
p.join()
print ("\nData normalization pipeline completed successfully ...")
else:
print ("\nFailed to initiate data normalization pipeline ...")
return False
return True
In [0]:
def generate_plot(args, name, model_train):
gen_plot = args.plot[0]
if gen_plot==True:
plot_training(args, name, model_train)
else:
print ("\nNo training summary plots generated ...")
print ("Set: --plot True for creating training summary plots")
In [0]:
def plot_training(args, name, history):
output_loc = args.output_dir[0]
output_file_acc = os.path.join(output_loc+
"//training_plot_acc_" +
timestr+str(name)+".png")
output_file_loss = os.path.join(output_loc+
"//training_plot_loss_" +
timestr+str(name)+".png")
fig_acc = plt.figure()
plt.plot(history.history['acc'])
plt.plot(history.history['val_acc'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
fig_acc.savefig(output_file_acc, dpi=fig_acc.dpi)
print ("Successfully created the training accuracy plot: "
+ str(output_file_acc))
plt.close()
fig_loss = plt.figure()
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
fig_loss.savefig(output_file_loss, dpi=fig_loss.dpi)
print ("Successfully created the loss function plot: "
+ str(output_file_loss))
plt.close()
In [0]:
def select_optimizer(args):
optimizer_val = args.optimizer_val[0]
lr = args.learning_rate[0]
decay = args.decay[0]
epsilon = args.epsilon[0]
rho = args.rho[0]
beta_1 = args.beta_1[0]
beta_2 = args.beta_2[0]
if optimizer_val.lower() == 'sgd' :
optimizer = SGD(lr=lr, \
decay=decay, \
momentum=1, \
nesterov=False)
print ("Using SGD as the optimizer ...")
elif optimizer_val.lower() == 'nsgd':
optimizer = SGD(lr=lr, \
decay=decay,\
momentum=1, \
nesterov=True)
print ("Using SGD as the optimizer with Nesterov momentum ...")
elif optimizer_val.lower() == 'rms' \
or \
optimizer_val.lower() == 'rmsprop':
optimizer = RMSprop(lr=lr, \
rho=rho, \
epsilon=epsilon,\
decay=decay)
print ("Using RMSProp as the optimizer ...")
elif optimizer_val.lower() == 'ada' \
or \
optimizer_val.lower() == 'adagrad':
optimizer = Adagrad(lr=lr, \
epsilon=epsilon, \
decay=decay)
print ("Using Adagrad as the optimizer ...")
elif optimizer_val.lower() == 'adelta' \
or \
optimizer_val.lower() == 'adadelta':
optimizer = Adadelta(lr=lr, \
rho=rho, \
epsilon=epsilon, \
decay=decay)
print ("Using Adadelta as the optimizer ...")
elif optimizer_val.lower() == 'adam':
optimizer = Adam(lr=lr, \
beta_1=beta_1, \
beta_2=beta_2, \
epsilon=epsilon, \
decay=decay, \
amsgrad=False)
print ("Using Adam as the optimizer ...")
print ("Optimizer parameters (recommended default): ")
print ("\n lr={} (0.001), \
\n beta_1={} (0.9), \
\n beta_2={} (0.999), \
\n epsilon={} (1e-08), \
\n decay={} (0.0)".format(lr,
beta_1,
beta_2,
epsilon,
decay))
elif optimizer_val.lower() == 'amsgrad':
optimizer = Adam(lr=lr, \
beta_1=beta_1, \
beta_2=beta_2, \
epsilon=epsilon, \
decay=decay, \
amsgrad=True)
print ("Using AmsGrad variant of Adam as the optimizer ...")
print ("Optimizer parameters (recommended default): ")
print ("\n lr={} (0.001), \
\n beta_1={} (0.9), \
\n beta_2={} (0.999), \
\n epsilon={} (1e-08), \
\n decay={} (0.0)".format(lr,
beta_1,
beta_2,
epsilon,
decay))
elif optimizer_val.lower() == 'adamax':
optimizer = Adamax(lr=lr, \
beta_1=beta_1, \
beta_2=beta_2, \
epsilon=epsilon, \
decay=decay)
print ("Using Adamax variant of Adam as the optimizer ...")
print ("Optimizer parameters (recommended default): ")
print ("\n lr={} (0.002), \
\n beta_1={} (0.9), \
\n beta_2={} (0.999), \
\n epsilon={} (1e-08), \
\n schedule_decay={} (0.0)".format(lr,
beta_1,
beta_2,
epsilon,
decay))
elif optimizer_val.lower() == 'nadam':
optimizer = Nadam(lr=lr, \
beta_1=beta_1, \
beta_2=beta_2, \
epsilon=epsilon, \
schedule_decay=decay)
print ("Using Nesterov Adam optimizer ...\
\n decay arguments is passed on to schedule_decay variable ...")
print ("Optimizer parameters (recommended default): ")
print ("\n lr={} (0.002), \
\n beta_1={} (0.9), \
\n beta_2={} (0.999), \
\n epsilon={} (1e-08), \
\n schedule_decay={} (0.004)".format(lr,
beta_1,
beta_2,
epsilon,
decay))
else:
optimizer = DEFAULT_OPTIMIZER
print ("Using stochastic gradient descent with Nesterov momentum ('nsgd') as the default optimizer ...")
print ("Options for optimizer are: 'sgd', \
\n'nsgd', \
\n'rmsprop', \
\n'adagrad', \
\n'adadelta', \
\n'adam', \
\n'nadam', \
\n'amsgrad', \
\n'adamax' ...")
return optimizer
In [0]:
def process_model(args,
model,
base_model,
optimizer,
loss,
checkpointer_savepath):
load_weights_ = args.load_weights[0]
fine_tune_model = args.fine_tune[0]
load_checkpoint = args.load_checkpoint[0]
if load_weights_ == True:
try:
with open(args.config_file[0]) as json_file:
model_json = json_file.read()
model = model_from_json(model_json)
except:
model = model
try:
model.load_weights(args.weights_file[0])
print ("\nLoaded model weights from: " + str(args.weights_file[0]))
except:
print ("\nError loading model weights ...")
print ("Tabula rasa ...")
print ("Loaded default model weights ...")
elif load_checkpoint == True and os.path.exists(checkpointer_savepath):
try:
model = load_model(checkpointer_savepath)
print ("\nLoaded model from checkpoint: " + str(checkpointer_savepath))
except:
if os.path.exists(args.saved_chkpnt[0]):
model = load_model(args.saved_chkpnt[0])
print ('\nLoaded saved checkpoint file ...')
else:
print ("\nError loading model checkpoint ...")
print ("Tabula rasa ...")
print ("Loaded default model weights ...")
else:
model = model
print ("\nTabula rasa ...")
print ("Loaded default model weights ...")
try:
NB_FROZEN_LAYERS = args.frozen_layers[0]
except:
NB_FROZEN_LAYERS = DEFAULT_NB_LAYERS_TO_FREEZE
if fine_tune_model == True:
print ("\nFine tuning Inception architecture ...")
print ("Frozen layers: " + str(NB_FROZEN_LAYERS))
model = finetune_model(model, optimizer, loss, NB_FROZEN_LAYERS)
else:
print ("\nTransfer learning using Inception architecture ...")
model = transferlearn_model(model, base_model, optimizer, loss)
return model
In [0]:
def process_images(args):
train_aug = args.train_aug[0]
test_aug = args.test_aug[0]
if str((args.base_model[0]).lower()) == 'inceptionv4' or \
str((args.base_model[0]).lower()) == 'inception_v4' or \
str((args.base_model[0]).lower()) == 'inception_resnet':
preprocess_input = preprocess_input_inceptionv4
else:
preprocess_input = preprocess_input_inceptionv3
if train_aug==True:
try:
train_rotation_range = args.train_rot[0]
train_width_shift_range = args.train_w_shift[0]
train_height_shift_range = args.train_ht_shift[0]
train_shear_range = args.train_shear[0]
train_zoom_range = args.train_zoom[0]
train_vertical_flip = args.train_vflip[0]
train_horizontal_flip = args.train_hflip[0]
except:
train_rotation_range = 30
train_width_shift_range = 0.2
train_height_shift_range = 0.2
train_shear_range = 0.2
train_zoom_range = 0.2
train_vertical_flip = True
train_horizontal_flip = True
print ("\nFailed to load custom training image augmentation parameters ...")
print ("Loaded pre-set defaults ...")
print ("To switch off image augmentation during training, set --train_augmentation flag to False")
train_datagen = ImageDataGenerator(preprocessing_function=preprocess_input,
rotation_range=train_rotation_range,
width_shift_range=train_width_shift_range,
height_shift_range=train_height_shift_range,
shear_range=train_shear_range,
zoom_range=train_zoom_range,
vertical_flip=train_vertical_flip,
horizontal_flip=train_horizontal_flip)
print ("\nCreated image augmentation pipeline for training images ...")
print ("Image augmentation parameters for training images: \
\n image rotation range = {},\
\n width shift range = {},\
\n height shift range = {}, \
\n shear range = {} ,\
\n zoom range = {}, \
\n enable vertical flip = {}, \
\n enable horizontal flip = {}".format(train_rotation_range,
train_width_shift_range,
train_height_shift_range,
train_shear_range,
train_zoom_range,
train_vertical_flip,
train_horizontal_flip))
else:
train_datagen = ImageDataGenerator(preprocessing_function=preprocess_input)
if test_aug==True:
try:
test_rotation_range = args.test_rot[0]
test_width_shift_range = args.test_w_shift[0]
test_height_shift_range = args.test_ht_shift[0]
test_shear_range = args.test_shear[0]
test_zoom_range = args.test_zoom[0]
test_vertical_flip = args.test_vflip[0]
test_horizontal_flip = args.test_hflip[0]
except:
test_rotation_range = 30
test_width_shift_range = 0.2
test_height_shift_range = 0.2
test_shear_range = 0.2
test_zoom_range = 0.2
test_vertical_flip = True
test_horizontal_flip = True
print ("\nFailed to load custom training image augmentation parameters ...")
print ("Loaded pre-set defaults ...")
print ("To switch off image augmentation during training, set --train_augmentation flag to False")
test_datagen = ImageDataGenerator(preprocessing_function=preprocess_input,
rotation_range=test_rotation_range,
width_shift_range=test_width_shift_range,
height_shift_range=test_height_shift_range,
shear_range=test_shear_range,
zoom_range=test_zoom_range,
vertical_flip=test_vertical_flip,
horizontal_flip=test_horizontal_flip)
print ("\nCreated image augmentation pipeline for training images ...")
print ("\nImage augmentation parameters for training images:")
print( "\n image rotation range = {},\
\n width shift range = {},\
\n height shift range = {}, \
\n shear range = {} ,\
\n zoom range = {}, \
\n enable vertical flip = {}, \
\n enable horizontal flip = {}".format(test_rotation_range,
test_width_shift_range,
test_height_shift_range,
test_shear_range,
test_zoom_range,
test_vertical_flip,
test_horizontal_flip))
else:
test_datagen = ImageDataGenerator(preprocessing_function=preprocess_input)
return [train_datagen, test_datagen]
In [0]:
def gen_model(args, enable_dropout):
if str((args.base_model[0]).lower()) == 'inceptionv4' or \
str((args.base_model[0]).lower()) == 'inception_v4' or \
str((args.base_model[0]).lower()) == 'inception_resnet':
base_model = InceptionResNetV2(weights='imagenet', \
include_top=False)
base_model_name = 'Inception version 4'
else:
base_model = InceptionV3(weights='imagenet',
include_top=False)
base_model_name = 'Inception version 3'
print ('\nBase model: ' + str(base_model_name))
nb_classes = len(glob.glob(args.train_dir[0] + "/*"))
model = add_top_layer(args,
enable_dropout,
base_model,
nb_classes)
print ("New top layer added to: " + str(base_model_name))
return [model, base_model]
In [0]:
def train(args):
"""
A function that takes the user arguments and initiates a training session of the neural network.
This function takes only one input: args
Example usage:
if train_model == True:
print ("Training sesssion initiated ...")
train(args)
"""
if not os.path.exists(args.output_dir[0]):
os.makedirs(args.output_dir[0])
optimizer = select_optimizer(args)
loss = args.loss[0]
checkpointer_savepath = os.path.join(args.output_dir[0] +
'/checkpoint/Transfer_learn_' +
str(IM_WIDTH) + '_' +
str(IM_HEIGHT) + '_' + '.h5')
nb_train_samples = get_nb_files(args.train_dir[0])
nb_classes = len(glob.glob(args.train_dir[0] + "/*"))
print ("\nTotal number of training samples = " + str(nb_train_samples))
print ("Number of training classes = " + str(nb_classes))
nb_val_samples = get_nb_files(args.val_dir[0])
nb_val_classes = len(glob.glob(args.val_dir[0] + "/*"))
print ("\nTotal number of validation samples = " + str(nb_val_samples))
print ("Number of validation classes = " + str(nb_val_classes))
if nb_val_classes == nb_classes:
print ("\nInitiating training session ...")
else:
print ("\nMismatched number of training and validation data classes ...")
print ("Unequal number of sub-folders found between train and validation directories ...")
print ("Each sub-folder in train and validation directroies are treated as a separate class ...")
print ("Correct this mismatch and re-run ...")
print ("\nNow exiting ...")
sys.exit(1)
nb_epoch = int(args.epoch[0])
batch_size = int(args.batch[0])
[train_datagen, validation_datagen] = process_images(args)
labels = generate_labels(args)
train_dir = args.train_dir[0]
val_dir = args.val_dir[0]
if args.normalize[0] and os.path.exists(args.root_dir[0]):
normalize(args,
labels,
move = False,
sub_sample = args.sub_sample[0])
train_dir = os.path.join(args.root_dir[0] +
str ('/.tmp_train/'))
val_dir = os.path.join(args.root_dir[0] +
str ('/.tmp_validation/'))
print ("\nGenerating training data: ... ")
train_generator = train_datagen.flow_from_directory(train_dir,
target_size=(IM_WIDTH, IM_HEIGHT),
batch_size=batch_size,
class_mode='categorical')
print ("\nGenerating validation data: ... ")
validation_generator = validation_datagen.flow_from_directory(val_dir,
target_size=(IM_WIDTH, IM_HEIGHT),
batch_size=batch_size,
class_mode='categorical')
[model, base_model] = gen_model(args, enable_dropout)
model = process_model(args,
model,
base_model,
optimizer,
loss,
checkpointer_savepath)
print ("\nInitializing training with class labels: " +
str(labels))
model_summary_ = args.model_summary[0]
if model_summary_ == True:
print (model.summary())
else:
print ("\nSuccessfully loaded deep neural network classifier for training ...")
print ("\nReady, Steady, Go ...")
print ("\n")
if not os.path.exists(os.path.join(args.output_dir[0] + '/checkpoint/')):
os.makedirs(os.path.join(args.output_dir[0] + '/checkpoint/'))
lr = args.learning_rate[0]
earlystopper = EarlyStopping(patience=6,
verbose=1)
checkpointer = ModelCheckpoint(checkpointer_savepath,
verbose=1,
save_best_only=True)
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=2,
mode = 'max',
epsilon=1e-4,
cooldown=1,
verbose=1,
factor=0.5,
min_lr=lr*1e-2)
model_train = model.fit_generator(train_generator,
epochs=nb_epoch,
steps_per_epoch=nb_train_samples//20,
validation_data=validation_generator,
validation_steps=nb_val_samples//20,
class_weight='auto',
callbacks=[earlystopper,
learning_rate_reduction,
checkpointer])
if args.fine_tune[0] == True:
save_model(args, "_ft_", model)
generate_plot(args, "_ft_", model_train)
else:
save_model(args, "_tl_", model)
generate_plot(args, "_tl_", model_train)
In [0]:
import types
args=types.SimpleNamespace()
args.base_model=['Inception_V4']
args.frozen_layers=[NB_FROZEN_LAYERS]
args.optimizer_val=['amsgrad']
args.decay=[0.0]
args.beta_2=[0.999]
args.beta_1=[0.9]
args.rho=[0.9]
args.learning_rate=[1e-3]
args.loss=['categorical_crossentropy']
args.activation=['sigmoid']
args.epsilon=[1e-8]
args.dropout=[0.4]
args.test_hflip=[True]
args.test_vflip=[True]
args.test_zoom=[True]
args.test_shear=[True]
args.train_model=[enable_training]
args.output_dir=['./{}/'.format(dataset_id)]
args.root_dir=['./']
args.val_dir=['./{}/validation/'.format(dataset_id)]
args.train_dir=['./{}/train/'.format(dataset_id)]
args.epoch=[20]
args.batch=[10]
args.train_aug=[True]
args.test_aug=[True]
args.normalize=[False]
args.sub_sample=[False]
args.load_weights=[False]
args.fine_tune=[True]
args.load_checkpoint=[True]
args.model_summary=[False]
args.plot=[True]
In [0]:
command = ['mkdir ./{}/validation/'.format(dataset_id),
'mkdir ./{}/validation/cats/'.format(dataset_id),
'mkdir ./{}/validation/dogs/'.format(dataset_id),
'cd ./{}/train/cats/ ; shuf -n 1000 -e * | xargs -i mv {} ../../../{}/validation/cats/'.format(dataset_id,
'{}',
dataset_id),
'cd ./{}/train/dogs/ ; shuf -n 1000 -e * | xargs -i mv {} ../../../{}/validation/dogs/'.format(dataset_id,
'{}',
dataset_id)]
In [0]:
if setup:
execute_in_shell(command = command,
verbose = True)
In [0]:
IM_WIDTH, IM_HEIGHT = 299, 299 # Default input image size for Inception v3 and v4 architecture
DEFAULT_EPOCHS = 100
DEFAULT_BATCHES = 20
FC_SIZE = 4096
DEFAULT_DROPOUT = 0.1
DEFAULT_NB_LAYERS_TO_FREEZE = 169
verbose = False
sgd = SGD(lr=1e-7, decay=0.5, momentum=1, nesterov=True)
rms = RMSprop(lr=1e-7, rho=0.9, epsilon=1e-08, decay=0.0)
ada = Adagrad(lr=1e-3, epsilon=1e-08, decay=0.0)
In [0]:
if enable_training:
train(args)
In [0]:
command = ['rm ./Transfer_learn_299_299_{}.h5'.format(dataset_id),
'cp ./{}/checkpoint/Transfer_learn_299_299_.h5 ./Transfer_learn_299_299_{}.h5'.format(dataset_id,
dataset_id)]
In [0]:
execute_in_shell(command = command,
verbose = True)
In [0]:
file_dir = './'
file_name = 'Transfer_learn_299_299_{}.h5'.format(dataset_id)
In [0]:
if colab_mode:
drive = cloud_authenticate()
In [0]:
if upload_weights and colab_mode:
googledrive_save(file_name = file_name,
file_dir = file_dir,
upload = True)
In [0]:
model = gen_model(args,
enable_dropout)[0]
In [0]:
model.load_weights('./Transfer_learn_299_299_{}.h5'.format(dataset_id))
In [0]:
model.summary()
In [0]:
command = ['apt-get install -y graphviz libgraphviz-dev && pip3 install pydot graphviz']
In [0]:
if setup:
execute_in_shell(command = command, verbose = True)
In [0]:
from keras.utils import plot_model
import pydot
import graphviz # apt-get install -y graphviz libgraphviz-dev && pip3 install pydot graphviz
from IPython.display import SVG
from keras.utils.vis_utils import model_to_dot
In [0]:
output_dir = './'
plot_model(model, to_file= output_dir + '/model_summary_plot.png')
SVG(model_to_dot(model).create(prog='dot', format='svg'))
In [0]:
! wget https://cdn-images-1.medium.com/max/1600/1*mONNI1lG9VuiqovpnYqicA.jpeg -O cats_01.jpeg
! wget https://www.petspyjamas.com/uploads/2013/07/can-cats-and-dogs-be-friends-6.jpg -O cat_and_dog_01.jpg
! wget https://github.com/rahulremanan/python_tutorial/raw/master/Machine_Vision/01_Transfer_Learning/media/goose_the_cat.png -O goose_the_cat.png
In [0]:
import cv2
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import keras.applications.inception_resnet_v2 as InceptionResNetV2
import tqdm
from keras import backend as K
from keras.layers import UpSampling2D, Conv2D
from keras.preprocessing import image
In [0]:
try:
labels_json='./dogs_vs_cats/trained_labels.json'
with open(labels_json) as json_file:
labels = json.load(json_file)
print (labels)
except:
labels = ["cats", "dogs"]
In [0]:
PRE_PROCESSOR = preprocess_input_inceptionv4
MODEL = model
INPUT_IMG_FILE = './goose_the_cat.png'
LABELS= labels
In [0]:
%matplotlib inline
In [0]:
img=mpimg.imread(INPUT_IMG_FILE)
plt.imshow(img)
In [0]:
import tensorflow as tf
import gc
In [0]:
def class_activation_map(INPUT_IMG_FILE=None,
PRE_PROCESSOR=None,
LABEL_DECODER=None,
MODEL=None,
LABELS=None,
IM_WIDTH=299,
IM_HEIGHT=299,
CONV_LAYER='conv_7b',
URL_MODE=False,
FILE_MODE=False,
EVAL_STEPS=10,
HEATMAP_SHAPE=[8,8],
BENCHMARK=True):
"""
A function to visualize class activation maps.
Also generate a Bayesian class activation map, that outputs a list of
heatmaps summarizing the model uncertainty.
Currently has performance scalability issues for number of evaluation steps,
due to the nature in which Tensorflow computes gradeints.
See the description of the problem here: https://stackoverflow.com/questions/36245481/tensorflow-slow-performance-when-getting-gradients-at-inputs
"""
#K.clear_session()
if INPUT_IMG_FILE == None:
print ('No input file specified to generate predictions ...')
return
if URL_MODE:
response = requests.get(INPUT_IMG_FILE)
img = Image.open(BytesIO(response.content))
img = img.resize((IM_WIDTH, IM_HEIGHT))
elif FILE_MODE:
img = INPUT_IMG_FILE
else:
img = image.load_img(INPUT_IMG_FILE, target_size=(IM_WIDTH, IM_HEIGHT))
x = img
if not FILE_MODE:
x = image.img_to_array(img)
x = np.expand_dims(x, axis=0)
if PRE_PROCESSOR !=None:
preprocess_input = PRE_PROCESSOR
x = preprocess_input(x)
model = MODEL
if model == None:
print ('No input model specified to generate predictions ...')
return
labels = LABELS
heatmaps = []
last_conv_layer = model.get_layer(CONV_LAYER)
feature_size = tensor_featureSizeExtractor(last_conv_layer)
model_input = model.input
model_output = model.output
last_conv_layer_out = last_conv_layer.output
iterate_input = []
pred_labels = []
out_labels = []
probabilities = np.empty((0,len(labels)), float)
for step in (range(EVAL_STEPS)):
input_img = x
startTime = time.time()
preds = model.predict(x, batch_size=1)
preds_endTime = time.time()
probability = preds.flatten()
probabilities = np.append(probabilities,
np.array([probability]),
axis=0)
if labels !=None:
pred_label = labels[np.argmax(probability)]
pred_labels.append(pred_label)
out_labels.append(pred_label)
print('PREDICTION: {}'.format(pred_label))
print('ACCURACY: {}'.format(preds[0]))
del pred_label
elif LABEL_DECODER !=None:
pred_label = pd.DataFrame(LABEL_DECODER(preds, top=3)[0],
columns=['col1',
'category',
'probability']).iloc[:,1:]
pred_labels.append(pred_label.loc[0,'category'])
out_labels.append(pred_label.loc[0,'category'])
print('PREDICTION:',pred_label.loc[0,'category'])
del pred_label
else:
print ('No labels will be generated ...')
pred_labels = set(pred_labels)
pred_labels = list(pred_labels)
argmax = np.argmax(probability)
heatmap_startTime = time.time()
output = model_output[:, argmax]
model_endTime = time.time()
grads = K.gradients(output,
last_conv_layer_out)[0]
pooled_grads = K.mean(grads,
axis=(0, 1, 2))
iterate = K.function([model_input], [pooled_grads,
last_conv_layer_out[0]])
pooled_grads_value, conv_layer_output_value = iterate([input_img])
grad_endTime = time.time()
for i in range(feature_size):
conv_layer_output_value[:,:,i] *= pooled_grads_value[i]
iter_endTime = time.time()
heatmap = np.mean(conv_layer_output_value, axis=-1)
heatmap = np.maximum(heatmap, 0)
heatmap /= np.max(heatmap)
heatmap_endTime = time.time()
try:
heatmaps.append(heatmap)
if EVAL_STEPS >1:
del probability
del heatmap
del output
del grads
del pooled_grads
del iterate
del pooled_grads_value
del conv_layer_output_value
del input_img
gc.collect()
except:
print ('Failed updating heatmaps ...')
endTime = time.time()
predsTime = preds_endTime - startTime
gradsTime = grad_endTime - model_endTime
iterTime = iter_endTime - grad_endTime
heatmapTime = heatmap_endTime - heatmap_startTime
executionTime = endTime - startTime
model_outputTime = model_endTime - heatmap_startTime
if BENCHMARK:
print ('Heatmap generation time: {} seconds ...'. format(heatmapTime))
print ('Gradient generation time: {} seconds ...'.format(gradsTime))
print ('Iteration loop execution time: {} seconds ...'.format(iterTime))
print ('Model output generation time: {} seconds'.format(model_outputTime))
print ('Prediction generation time: {} seconds ...'.format(predsTime))
print ('Completed processing {} out of {} steps in {} seconds ...'.format(int(step+1), int(EVAL_STEPS), float(executionTime)))
print ('\n')
print ('Percentage time spent generating heatmap: {}'.format((heatmapTime/executionTime)*100))
print ('Percentage time spent generating gradients: {}'.format((gradsTime/executionTime)*100))
print ('Percentage time spent generating iteration loop: {}'.format((iterTime/executionTime)*100))
print ('Percentage time spent generating model outputs: {}'.format((model_outputTime/executionTime)*100))
print ('Percentage time spent generating predictions: {}'.format((predsTime/executionTime)*100))
print ('\n')
if EVAL_STEPS >1:
heatmap_sum = heatmaps[0]
for i in range(len(heatmaps)-1):
if i<= len(heatmaps):
heatmap_sum = np.nan_to_num(heatmaps[i+1])+np.nan_to_num(heatmap_sum)
print (heatmap_sum)
mean_heatmap = heatmap_sum/len(heatmaps)
else:
mean_heatmap = heatmap
mean = np.matrix.mean(np.asmatrix(probabilities), axis=0)
stdev = np.matrix.std(np.asmatrix(probabilities), axis=0)
accuracy = np.matrix.tolist(mean)[0][np.argmax(mean)]
uncertainty = np.matrix.tolist(stdev)[0][np.argmax(mean)]
return [mean_heatmap, accuracy, uncertainty, pred_labels, heatmaps, out_labels, probabilities]
In [0]:
def tensor_featureSizeExtractor(last_conv_layer):
if len(last_conv_layer.output.get_shape().as_list()) == 4:
feature_size = last_conv_layer.output.get_shape().as_list()[3]
return feature_size
else:
print ('Received tensor shape: {} instead of expected shape: 4'.format(len(last_conv_layer.output.get_shape().as_list())))
return None
In [0]:
def heatmap_overlay(INPUT_IMG_FILE,
HEATMAP,
THRESHOLD=0.8):
img = cv2.imread(INPUT_IMG_FILE)
heatmap = cv2.resize(HEATMAP, (img.shape[1], img.shape[0]))
heatmap = np.uint8(255 * heatmap)
heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)
hif = THRESHOLD
#superimposed_img = heatmap * hif + img
superimposed_img = cv2.addWeighted(img,THRESHOLD,heatmap,1-THRESHOLD,0)
return [superimposed_img, heatmap]
In [0]:
if bayesian_cam:
output = class_activation_map(INPUT_IMG_FILE=INPUT_IMG_FILE,
PRE_PROCESSOR=PRE_PROCESSOR,
MODEL=MODEL,
LABELS=LABELS,
IM_WIDTH=299,
IM_HEIGHT=299,
CONV_LAYER='conv_7b',
EVAL_STEPS=100)
HEATMAP = output[0]
plt.matshow(HEATMAP)
plt.show()
print (output[3])
In [0]:
if bayesian_cam:
heatmap_output = heatmap_overlay(INPUT_IMG_FILE,
HEATMAP,
THRESHOLD=0.8)
superimposed_img = heatmap_output[0]
In [0]:
if bayesian_cam:
output_file = './class_activation_map.jpeg'
cv2.imwrite(output_file, superimposed_img)
img=mpimg.imread(output_file)
In [0]:
if bayesian_cam:
plt.imshow(img)
In [0]:
import cv2
import numpy as np
import glob
if bayesian_cam:
heatmaps=output[4]
labels=output[5]
img_array = []
for i in range(len(heatmaps)):
HEATMAP = heatmaps[i]
LABEL = labels[i]
heatmap_output = heatmap_overlay(INPUT_IMG_FILE,
HEATMAP,
THRESHOLD=0.7)
height, width, layers = heatmap_output[0].shape
size = (width,height)
superimposed_img = heatmap_output[0]
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.putText(superimposed_img,'{}'.format(LABEL),(10,100), font, 4,(255,255,255),2)
img_array.append(np.uint8(superimposed_img))
out = cv2.VideoWriter('bayesian_class_activation_maps.avi',cv2.VideoWriter_fourcc(*'DIVX'), 8, size)
for i in range(len(img_array)):
out.write(img_array[i])
out.release()
In [0]:
! apt-get install handbrake handbrake-cli
In [0]:
! HandBrakeCLI -i ./bayesian_class_activation_maps.avi -o ./bayesian_class_activation_maps.mp4 -e x264 -q 22 -r 15 -B 64 -X 480 -O
In [0]:
import io
import base64
from IPython.display import HTML
video = io.open('./bayesian_class_activation_maps.mp4', 'r+b').read()
encoded = base64.b64encode(video)
HTML(data='''<video alt="test" controls>
<source src="data:video/mp4;base64,{0}" type="video/mp4" />
</video>'''.format(encoded.decode('ascii')))
In [0]:
download_output=True
from google.colab import files
if download_output and bayesian_cam:
files.download('./bayesian_class_activation_maps.mp4')
files.download('./class_activation_map.jpeg')
files.download('./model_summary_plot.png')
In [0]:
import requests
from io import BytesIO
from PIL import Image
from keras.preprocessing import image
from keras.applications.inception_resnet_v2 import preprocess_input
In [0]:
target_size = (IM_WIDTH, IM_HEIGHT)
INPUT_IMAGE="./dogs_vs_cats/test/test/4106.jpg"
In [0]:
from IPython.display import Image as PyImage
PyImage(INPUT_IMAGE)
In [0]:
img = Image.open(INPUT_IMAGE)
if img.size != target_size:
img = img.resize((299, 299))
_x_ = image.img_to_array(img)
_x_ = np.expand_dims(_x_, axis=0)
_x_ = preprocess_input(_x_)
In [0]:
model.predict(_x_)
In [0]:
def plot_preds(preds, labels, timestr):
output_loc = args.output_dir[0]
output_file_preds = os.path.join(output_loc+"//preds_out_"+timestr+".png")
fig = plt.figure()
plt.axis('on')
labels = labels
plt.barh([0, 1], preds, alpha=0.5)
plt.yticks([0, 1], labels)
plt.xlabel('Probability')
plt.xlim(0,1.01)
plt.tight_layout()
fig.savefig(output_file_preds, dpi=fig.dpi)
In [0]:
try:
labels_json='./dogs_vs_cats/trained_labels.json'
with open(labels_json) as json_file:
labels = json.load(json_file)
print (labels)
except:
labels = ["cats", "dogs"]
In [0]:
def predict(model=None,
img=None,
labels=None,
target_size=None,
bayesian_inference=True,
eval_steps=2,
detection_threshold=0.5,
prediction_max=True,
verbose=False):
if verbose:
print ("Running prediction model on the image file ...")
if img.size != target_size:
img = img.resize(target_size)
_x_ = image.img_to_array(img)
_x_ = np.expand_dims(_x_, axis=0)
_x_ = preprocess_input(_x_)
probabilities = np.empty((0,len(labels)), float)
if bayesian_inference:
for step in (range(eval_steps)):
start = time.time()
preds = model.predict(_x_, batch_size=1)
probability = preds.flatten()
probabilities = np.append(probabilities, np.array([probability]), axis=0)
end = time.time()
execution_time = end - start
if verbose:
print ('Execution time: {} seconds ...'.format(execution_time))
mean = np.matrix.mean(np.asmatrix(probabilities), axis=0)
max_ = np.matrix.max(np.asmatrix(probabilities), axis=0)
if prediction_max:
out = max_
else:
out = mean
predictions = np.matrix.tolist(out)[0]
uncertainty = np.matrix.std(np.asmatrix(probabilities), axis=0)
uncertainty = np.matrix.tolist(uncertainty)[0][np.argmax(predictions)]
preds_label = []
for i in range(len(labels)):
if predictions[i] >= detection_threshold:
preds_label.append(labels[i])
accuracy = predictions[np.argmax(predictions)]
probabilities = np.matrix.tolist(probabilities)[0]
return predictions, preds_label, accuracy, uncertainty, max_, probabilities
else:
preds = model.predict(_x_, batch_size=1)
probability = preds.flatten()
preds_label = labels[np.argmax(probability)]
accuracy = probability[np.argmax(probability)]
return preds[0], preds_label, accuracy, probability
In [0]:
if True:
img = Image.open(INPUT_IMAGE)
bayesian_inference=True
preds = predict(model,
img,
labels,
target_size,
bayesian_inference,
eval_steps=100,
detection_threshold=0.5,
prediction_max=False,
verbose=False)
print (preds[1])
print (preds[0])
print ('This picture contain: {}'.format(preds[1]))
print ('Predicted with accuracy of: {}'.format(preds[2]))
if bayesian_inference:
try:
print ('Uncertainty in prediction: {}'.format(preds[3]))
except:
print ('Uncertainty in prediction: {}'.format('Unknown'))
timestr = generate_timestamp()
plot_preds(preds[0], labels, timestr)
In [0]:
import requests
from io import BytesIO
from PIL import Image
from keras.preprocessing import image
bayesian_inference=True
In [0]:
args.image_url = ['https://cdn-images-1.medium.com/max/1600/1*mONNI1lG9VuiqovpnYqicA.jpeg']
target_size = (IM_WIDTH, IM_HEIGHT)
In [0]:
from IPython.display import Image as PyImage
from IPython.core.display import HTML
PyImage(url = args.image_url[0])
In [0]:
if args.image_url is not None:
response = requests.get(args.image_url[0])
img = Image.open(BytesIO(response.content))
preds = predict(model=model,
img=img,
labels=labels,
target_size=target_size,
bayesian_inference=True,
eval_steps=50,
detection_threshold=0.5,
prediction_max=False)
print (preds[1])
print (preds[0])
print ('This picture contain: {}'.format(preds[1]))
print ('Predicted with accuracy of: {}'.format(preds[2]))
if bayesian_inference:
try:
print ('Uncertainty in prediction: {}'.format(preds[3]))
except:
print ('Uncertainty in prediction: {}'.format('Unknown'))
timestr = generate_timestamp()
plot_preds(preds[0], labels, timestr)
In [0]:
#from IPython.display import Image as PyImage
#PyImage("./dogs_vs_cats/preds_out_{}.png".format(timestr))
In [0]:
args.image_url=['http://static.cdnbridge.com/resources/18/160536/picture/16/85388054.jpg']
In [0]:
PyImage(url = args.image_url[0])
In [0]:
if args.image_url is not None:
response = requests.get(args.image_url[0])
img = Image.open(BytesIO(response.content))
preds = predict(model=model,
img=img,
labels=labels,
target_size=target_size,
bayesian_inference=True,
eval_steps=50,
detection_threshold=0.5,
prediction_max=False)
print (preds[1])
print (preds[0])
print ('This picture contain: {}'.format(preds[1]))
print ('Predicted with accuracy of: {}'.format(preds[2]))
if bayesian_inference:
try:
print ('Uncertainty in prediction: {}'.format(preds[3]))
except:
print ('Uncertainty in prediction: {}'.format('Unknown'))
timestr = generate_timestamp()
plot_preds(preds[0], labels, timestr)
In [0]:
#from IPython.display import Image as PyImage
#PyImage("./dogs_vs_cats/preds_out_{}.png".format(timestr))
In [0]:
args.image_url=['https://www.shelterluv.com/sites/default/files/animal_pics/3451/2019/02/13/08/20190213083008.png']
In [0]:
from IPython.display import Image as PyImage
from IPython.core.display import HTML
PyImage(url = args.image_url[0])
In [0]:
if args.image_url is not None:
response = requests.get(args.image_url[0])
img = Image.open(BytesIO(response.content))
preds = predict(model,
img,
labels,
target_size,
bayesian_inference,
eval_steps=50,
detection_threshold=0.5,
prediction_max=False)
print (preds[1])
print (preds[0])
print ('This picture contain: {}'.format(preds[1]))
print ('Predicted with accuracy of: {}'.format(preds[2]))
if bayesian_inference:
try:
print ('Uncertainty in prediction: {}'.format(preds[3]))
except:
print ('Uncertainty in prediction: {}'.format('Unknown'))
timestr = generate_timestamp()
plot_preds(preds[0], labels, timestr)
In [0]:
#from IPython.display import Image as PyImage
#PyImage("./dogs_vs_cats/preds_out_{}.png".format(timestr))
In [0]:
args.image_url=['https://www.petspyjamas.com/uploads/2013/07/can-cats-and-dogs-be-friends-6.jpg']
In [0]:
from IPython.display import Image as PyImage
from IPython.core.display import HTML
PyImage(url = args.image_url[0])
In [0]:
if args.image_url is not None:
response = requests.get(args.image_url[0])
img = Image.open(BytesIO(response.content))
preds = predict(model,
img,
labels,
target_size,
bayesian_inference,
eval_steps=10,
detection_threshold=0.75,
prediction_max=False,
)
print (preds[1])
print (preds[0])
print ('This picture contain: {}'.format(preds[1]))
print ('Predicted with accuracy of: {}'.format(preds[2]))
if bayesian_inference:
try:
print ('Uncertainty in prediction: {}'.format(preds[3]))
except:
print ('Uncertainty in prediction: {}'.format('Unknown'))
timestr = generate_timestamp()
plot_preds(preds[0], labels, timestr)
In [0]:
#from IPython.display import Image as PyImage
#PyImage("./dogs_vs_cats/preds_out_{}.png".format(timestr))
In [0]:
rotation_range = 30
width_shift_range = 0.2
height_shift_range = 0.2
shear_range = 0.2
zoom_range = 0.2
vertical_flip = True
horizontal_flip = True
img_width = 299
img_height = 299
TEST_DIR = './dogs_vs_cats/test/test/'
OUTPUT_FILE = "./dogs_vs_cats_InceptionResNetV2.csv"
In [0]:
import re
import gc
from tqdm import tqdm
In [0]:
id_ = []
pred_labels = []
verbose = False
for i in tqdm(os.listdir(TEST_DIR)):
start = time.time()
img = Image.open(os.path.join(TEST_DIR+i))
preds = predict(model=model,
img=img,
labels=labels,
target_size=target_size,
bayesian_inference=True,
eval_steps=1,
detection_threshold=0.75,
prediction_max=False)
try:
pred_ = np.argmax(preds[0])
if pred_ == 0 and preds[0][pred_]<0.5:
pred_label = preds[0][pred_]
elif pred_ == 0 and preds[0][pred_]>0.5:
pred_label = 1-preds[0][pred_]
elif pred_ == 1 and preds[0][pred_]<0.5:
pred_label = 1-preds[0][pred_]
elif pred_ == 1 and preds[0][pred_]>0.5:
pred_label = preds[0][pred_]
else:
pred_label = 0.5
except:
pred_label = 0.5
id_.append(i)
pred_labels.append(pred_label)
end = time.time()
if verbose:
print ('Step: {} out of: {} in: {} seconds'.format(len(id_),
len(os.listdir(TEST_DIR)),
(end-start)))
print ('Completed generating predictions for: {} as: {}'.format(i,
pred_label))
solution = pd.DataFrame({"id":id_,
"label":pred_labels})
cols = ['label']
for col in cols:
solution[col] = solution[col].map(lambda x: str(x).lstrip('[').rstrip(']')).astype(str)
solution.to_csv(OUTPUT_FILE,
index = False)
In [0]:
from google.colab import files
files.download(OUTPUT_FILE)
In [0]: