Experiments approximating the posterior with diagonal Gaussians using variational inference

We start by building the model and showing the basic inference procedure and calculation of the performance on the MNIST classification and the outlier detection task. Then perform multiple runs of the model with different number of samples in the ensemble to calculate performance statistics. This experiment uses the same learning rate schedule as the SGLD example for comparable results.



In [1]:

    
# Let's first setup the libraries, session and experimental data
import experiment
import inferences
import edward as ed
import tensorflow as tf
import numpy as np
import os

s = experiment.setup()
mnist, notmnist = experiment.get_data()









    



Extracting MNIST_data/train-images-idx3-ubyte.gz
Extracting MNIST_data/train-labels-idx1-ubyte.gz
Extracting MNIST_data/t10k-images-idx3-ubyte.gz
Extracting MNIST_data/t10k-labels-idx1-ubyte.gz
Extracting notMNIST_data/train-images-idx3-ubyte.gz
Extracting notMNIST_data/train-labels-idx1-ubyte.gz
Extracting notMNIST_data/t10k-images-idx3-ubyte.gz
Extracting notMNIST_data/t10k-labels-idx1-ubyte.gz



In [2]:

    
# Builds the model and approximation variables used for the model
y_, model_variables = experiment.get_model_3layer()
approx_variables = experiment.get_vi_approximation_variables_3layer()



In [3]:

    
# Performs inference with edward's KLpq class
lr = tf.placeholder(tf.float32, shape=[])
optimizer = tf.train.GradientDescentOptimizer(lr)
inference_dict = {model_variables[key]: val for key, val in approx_variables.iteritems()}
inference = ed.KLpq(inference_dict, data={y_: model_variables['y']})
n_iter=1000
inference.initialize(n_iter=n_iter, optimizer=optimizer, n_samples=10)

tf.global_variables_initializer().run()
for i in range(n_iter):
    batch = mnist.train.next_batch(100)
    info_dict = inference.update({model_variables['x']: batch[0],
                                  model_variables['y']: batch[1],
                                  lr:0.005/(i+1.)})
    inference.print_progress(info_dict)

inference.finalize()









    



1000/1000 [100%] ██████████████████████████████ Elapsed: 59s | Loss: -27991.213



In [4]:

    
# Computes the accuracy of our model
accuracy, disagreement = experiment.get_metrics(model_variables, approx_variables, num_samples=10)
print(accuracy.eval({model_variables['x']: mnist.test.images, model_variables['y']: mnist.test.labels}))
print(disagreement.eval({model_variables['x']: mnist.test.images, model_variables['y']: mnist.test.labels}))









    



0.1087
[ 16.95742416  18.34371948  16.09437752 ...,  17.48067284  20.25243759
  14.18485832]



In [5]:

    
# Computes some statistics for the proposed outlier detection
outlier_stats = experiment.get_outlier_stats(model_variables, disagreement, mnist, notmnist)
print(outlier_stats)
print('TP/(FN+TP): {}'.format(float(outlier_stats['TP']) / (outlier_stats['TP'] + outlier_stats['FN'])))
print('FP/(FP+TN): {}'.format(float(outlier_stats['FP']) / (outlier_stats['FP'] + outlier_stats['TN'])))









    



{'FP': 0, 'TN': 10000, 'FN': 10000, 'TP': 0}
TP/(FN+TP): 0.0
FP/(FP+TN): 0.0

The following cell performs multiple runs of this model with different number of samples within the ensemble to capture performance statistics. Results are saved in VI_KLpq_SGLD_LR.csv.



In [6]:

    
import pandas as pd

results = pd.DataFrame(columns=('run', 'samples', 'acc', 'TP', 'FN', 'TN', 'FP'))

for run in range(5):
    lr = tf.placeholder(tf.float32, shape=[])
    optimizer = tf.train.GradientDescentOptimizer(lr)
    inference_dict = {model_variables[key]: val for key, val in approx_variables.iteritems()}
    inference = ed.KLpq(inference_dict, data={y_: model_variables['y']})
    n_iter=1000
    inference.initialize(n_iter=n_iter, optimizer=optimizer, n_samples=10)

    tf.global_variables_initializer().run()
    for i in range(n_iter):
        batch = mnist.train.next_batch(100)
        info_dict = inference.update({model_variables['x']: batch[0],
                                      model_variables['y']: batch[1],
                                      lr:0.005/(i+1.)})
        inference.print_progress(info_dict)

    inference.finalize()
    
    for num_samples in range(15):
        accuracy, disagreement = experiment.get_metrics(model_variables, approx_variables,
                                                        num_samples=num_samples + 1)
        acc = accuracy.eval({model_variables['x']: mnist.test.images, model_variables['y']: mnist.test.labels})
        outlier_stats = experiment.get_outlier_stats(model_variables, disagreement, mnist, notmnist)
        results.loc[len(results)] = [run, num_samples + 1, acc,
                                     outlier_stats['TP'], outlier_stats['FN'],
                                     outlier_stats['TN'], outlier_stats['FP']]
results.to_csv('VI_KLpq_SGLD_LR.csv', index=False)









    



1000/1000 [100%] ██████████████████████████████ Elapsed: 60s | Loss: -25997.809
1000/1000 [100%] ██████████████████████████████ Elapsed: 61s | Loss: -23620.547
1000/1000 [100%] ██████████████████████████████ Elapsed: 62s | Loss: -19648.000
1000/1000 [100%] ██████████████████████████████ Elapsed: 63s | Loss: -29080.588
1000/1000 [100%] ██████████████████████████████ Elapsed: 65s | Loss: -28918.875