bayesian linear regression



In [1]:

    
from edward.models import Normal
import tensorflow as tf
import matplotlib.pyplot as plt
import edward as ed
import seaborn as sns
import matplotlib
import matplotlib.cm as cm
from matplotlib.colors import ListedColormap, Normalize
from matplotlib.animation import FuncAnimation

matplotlib.interactive(True)
plt.style.use('seaborn-talk')

sns.palplot(sns.color_palette())



In [2]:

    
sess = ed.get_session()
ed.set_seed(42)



In [3]:

    
x= tf.range(-5.0,5.0,0.001)
plt.plot(*sess.run([x, Normal(loc=tf.ones(1) * 0.0,
                            scale =tf.ones(1)*1.0).prob(x)]))
plt.plot(*sess.run([x, Normal(loc=tf.ones(1)*2.0,
                             scale=tf.ones(1)*1.0).prob(x)]))
plt.plot(*sess.run([x, Normal(loc=tf.ones(1)*0.0,
                             scale=tf.ones(1)*1.0).prob(x)]))









    Out[3]:





[<matplotlib.lines.Line2D at 0x1143e0e10>]



In [9]:

    
# data
N1 = 10 # number of training data points in first batch 
N2 = 90 # number of training data points in second batch
Np = 10 # number of test data points
D = 1 # number of features

weights_true = sess.run(Normal(loc=tf.ones(D) * 2.0,
                              scale=tf.ones(D) * 0.1)) # unknown true weights
intercept_true = sess.run(Normal(loc=tf.zeros(1),
                                scale=tf.ones(1)))
noise_true = 0.35 # unknown true amount of noise

def build_dataset(N):
    x = Normal(loc=tf.zeros([N,D]), scale=tf.ones([N,D]))
    y = Normal(loc=ed.dot(x,weights_true) + intercept_true, scale=noise_true)
    return sess.run([x,y])

x_train1, y_train1 = build_dataset(N1)
x_train2, y_train2 = build_dataset(N2)
x_test, y_test = build_dataset(Np)



In [11]:

    
plt.scatter(x_train1, y_train1, s=20.0)
plt.scatter(x_test, y_test,s=20.0,color=sns.color_palette().as_hex()[2])









    Out[11]:





<matplotlib.collections.PathCollection at 0x115e9dfd0>

Little noise



In [18]:

    
from edward.models import Empirical, Normal
import numpy as np



In [22]:

    
# Forward model
x = tf.placeholder(tf.float32,[N1,D])
weights = Normal(loc=tf.zeros(D),scale=tf.ones(D))
intercept = Normal(loc=tf.zeros(1), scale=tf.ones(1))
y = Normal(loc=ed.dot(x,weights)+ intercept,
          scale=tf.ones(N1) ) # with more noise



In [23]:

    
# BACKWARD MODEL
T = 10000
q_weights = Empirical(params=tf.Variable(tf.zeros([T,D])))
q_intercept = Empirical(params=tf.Variable(tf.zeros([T,1])))



In [24]:

    
# inference 
inference = ed.HMC(latent_vars={weights: q_weights,
                                intercept: q_intercept},
                   data={x: x_train1,
                         y: y_train1})
inference.run(step_size=0.01 / N1, n_steps=20)









    



10000/10000 [100%] ██████████████████████████████ Elapsed: 99s | Acceptance Rate: 1.000



In [25]:

    
# criticism
plt.scatter(x_train1, y_train1, s=20.0)
plt.scatter(x_test, y_test, s=20.0, color = sns.color_palette().as_hex()[2])

xp = tf.placeholder(tf.float32, [2,D])
[plt.plot(np.linspace(-4.0,4.0,2),
         sess.run(ed.dot(xp, q_weights.params[t]) + q_intercept.params[t],
                 {xp:np.linspace(-4.0,4.0,2)[:,np.newaxis]}),
         color='k',alpha=0.1)
for t in range(int(T/2),T,int(T/100))]









    Out[25]:





[[<matplotlib.lines.Line2D at 0x1256f4d30>],
 [<matplotlib.lines.Line2D at 0x1240873c8>],
 [<matplotlib.lines.Line2D at 0x123fe3128>],
 [<matplotlib.lines.Line2D at 0x1241d63c8>],
 [<matplotlib.lines.Line2D at 0x1241cdeb8>],
 [<matplotlib.lines.Line2D at 0x1243ac9b0>],
 [<matplotlib.lines.Line2D at 0x1244bff98>],
 [<matplotlib.lines.Line2D at 0x1244efe48>],
 [<matplotlib.lines.Line2D at 0x124825e48>],
 [<matplotlib.lines.Line2D at 0x124adcc18>],
 [<matplotlib.lines.Line2D at 0x124e10e10>],
 [<matplotlib.lines.Line2D at 0x124e217f0>],
 [<matplotlib.lines.Line2D at 0x124e30e80>],
 [<matplotlib.lines.Line2D at 0x124fb0cf8>],
 [<matplotlib.lines.Line2D at 0x1251eeb38>],
 [<matplotlib.lines.Line2D at 0x1240bdb70>],
 [<matplotlib.lines.Line2D at 0x1240eef60>],
 [<matplotlib.lines.Line2D at 0x124025dd8>],
 [<matplotlib.lines.Line2D at 0x124139668>],
 [<matplotlib.lines.Line2D at 0x1241f9a58>],
 [<matplotlib.lines.Line2D at 0x124229ef0>],
 [<matplotlib.lines.Line2D at 0x1242463c8>],
 [<matplotlib.lines.Line2D at 0x12423ef60>],
 [<matplotlib.lines.Line2D at 0x1241179b0>],
 [<matplotlib.lines.Line2D at 0x12412cf98>],
 [<matplotlib.lines.Line2D at 0x12405de48>],
 [<matplotlib.lines.Line2D at 0x11de94e48>],
 [<matplotlib.lines.Line2D at 0x12417ec18>],
 [<matplotlib.lines.Line2D at 0x1241b3e10>],
 [<matplotlib.lines.Line2D at 0x11df977f0>],
 [<matplotlib.lines.Line2D at 0x11dfa6e80>],
 [<matplotlib.lines.Line2D at 0x11e022cf8>],
 [<matplotlib.lines.Line2D at 0x11df5db38>],
 [<matplotlib.lines.Line2D at 0x11df76b70>],
 [<matplotlib.lines.Line2D at 0x11df1df60>],
 [<matplotlib.lines.Line2D at 0x11dfdadd8>],
 [<matplotlib.lines.Line2D at 0x11dfee668>],
 [<matplotlib.lines.Line2D at 0x11dee9a58>],
 [<matplotlib.lines.Line2D at 0x11fba1ef0>],
 [<matplotlib.lines.Line2D at 0x11fbbb3c8>],
 [<matplotlib.lines.Line2D at 0x11fbb3f60>],
 [<matplotlib.lines.Line2D at 0x11e0509b0>],
 [<matplotlib.lines.Line2D at 0x11e062f98>],
 [<matplotlib.lines.Line2D at 0x11fd09e48>],
 [<matplotlib.lines.Line2D at 0x11fd37e48>],
 [<matplotlib.lines.Line2D at 0x1208e9c18>],
 [<matplotlib.lines.Line2D at 0x11e0abe10>],
 [<matplotlib.lines.Line2D at 0x11e0bd7f0>],
 [<matplotlib.lines.Line2D at 0x12087fe80>],
 [<matplotlib.lines.Line2D at 0x1208b9cf8>]]



In [26]:

    
y_post = ed.copy(y, {weights: q_weights, 
                    intercept: q_intercept})



In [27]:

    
print('Mean squared error on test data:')
print(ed.evaluate('mean_squared_error', data ={x: x_test, y_post: y_test}))

print("mean absolute error on test data")
print(ed.evaluate('mean_absolute_error', data={x: x_test, y_post: y_test}))









    



Mean squared error on test data:
0.111212
mean absolute error on test data
0.338317



In [28]:

    
# prior on noise



In [ ]:

    
from edward.models import InverseGamma



In [ ]:



In [ ]:



In [ ]: