We start by building the model and showing the basic inference procedure and calculation of the performance on the MNIST classification and the outlier detection task. Then perform multiple runs of the model with different number of samples in the ensemble to calculate performance statistics. Instead of SGLD we use an adaptation of Adam which is based on ideas from SGLD. So we add noise to the parameter updates in the order adaptive learning rate.
In [1]:
# Let's first setup the libraries, session and experimental data
import experiment
import inferences
import edward as ed
import tensorflow as tf
import numpy as np
import os
s = experiment.setup()
mnist, notmnist = experiment.get_data()
In [2]:
# Builds the model and approximation variables used for the model
y_, model_variables = experiment.get_model_3layer()
approx_variables = experiment.get_gauss_approximation_variables_3layer()
In [3]:
# Performs inference with our custom inference class
inference_dict = {model_variables[key]: val for key, val in approx_variables.iteritems()}
inference = inferences.VariationalGaussNoisyAdam(inference_dict, data={y_: model_variables['y']})
n_iter=1000
inference.initialize(n_iter=n_iter, learning_rate=0.005, burn_in=0)
tf.global_variables_initializer().run()
for i in range(n_iter):
batch = mnist.train.next_batch(100)
info_dict = inference.update({model_variables['x']: batch[0],
model_variables['y']: batch[1]})
inference.print_progress(info_dict)
inference.finalize()
In [4]:
# Computes the accuracy of our model
accuracy, disagreement = experiment.get_metrics(model_variables, approx_variables, num_samples=10)
print(accuracy.eval({model_variables['x']: mnist.test.images, model_variables['y']: mnist.test.labels}))
print(disagreement.eval({model_variables['x']: mnist.test.images, model_variables['y']: mnist.test.labels}))
In [5]:
# Computes some statistics for the proposed outlier detection
outlier_stats = experiment.get_outlier_stats(model_variables, disagreement, mnist, notmnist)
print(outlier_stats)
print('TP/(FN+TP): {}'.format(float(outlier_stats['TP']) / (outlier_stats['TP'] + outlier_stats['FN'])))
print('FP/(FP+TN): {}'.format(float(outlier_stats['FP']) / (outlier_stats['FP'] + outlier_stats['TN'])))
The following cell performs multiple runs of this model with different number of samples within the ensemble to capture performance statistics. Results are saved in SampledGauss_noisyAdam.csv
.
In [6]:
import pandas as pd
results = pd.DataFrame(columns=('run', 'samples', 'acc', 'TP', 'FN', 'TN', 'FP'))
for run in range(5):
inference_dict = {model_variables[key]: val for key, val in approx_variables.iteritems()}
inference = inferences.VariationalGaussNoisyAdam(inference_dict, data={y_: model_variables['y']})
n_iter=1000
inference.initialize(n_iter=n_iter, learning_rate=0.005, burn_in=0)
tf.global_variables_initializer().run()
for i in range(n_iter):
batch = mnist.train.next_batch(100)
info_dict = inference.update({model_variables['x']: batch[0],
model_variables['y']: batch[1]})
inference.print_progress(info_dict)
inference.finalize()
for num_samples in range(15):
accuracy, disagreement = experiment.get_metrics(model_variables, approx_variables,
num_samples=num_samples + 1)
acc = accuracy.eval({model_variables['x']: mnist.test.images, model_variables['y']: mnist.test.labels})
outlier_stats = experiment.get_outlier_stats(model_variables, disagreement, mnist, notmnist)
results.loc[len(results)] = [run, num_samples + 1, acc,
outlier_stats['TP'], outlier_stats['FN'],
outlier_stats['TN'], outlier_stats['FP']]
results.to_csv('SampledGauss_noisyAdam.csv', index=False)