In [1]:
import sys
sys.path.append("../")
import numpy as np
import tables
import cPickle as pickle
import theano
import theano.tensor as T
from learning.isb import ISB
from learning.dataset import MNIST
In [2]:
figsize(*figaspect(0.2))
def display_fields(F, n_fields=24, D=28):
for i in xrange(n_fields):
subplot(3, 8, i+1)
axis('off'); gray()
imshow(F[i].reshape( (D,D)), interpolation='nearest')
In [58]:
result_dir = "/home/joerg/LISA/importance-sampling-backprop/param-mnist-100-best/"
hparams = {}
execfile(result_dir+"/paramfile.py", hparams)
with tables.openFile(result_dir+"/results.h5", "r") as h5:
Lp_10 = h5.root.Lp_10[:]
Lp_25 = h5.root.Lp_25[:]
Lp_100 = h5.root.Lp_100[:]
Lp_500 = h5.root.Lp_100[:]
mparams = {
'P_a' : h5.root.P_a[-1,:],
'P_b' : h5.root.P_b[-1,:],
'P_W' : h5.root.P_W[-1,:,:],
'Q_c' : h5.root.Q_c[-1,:],
'Q_b' : h5.root.Q_b[-1,:],
'Q_W' : h5.root.Q_W[-1,:,:],
'Q_V' : h5.root.Q_V[-1,:,:],
'Q_Ub' : h5.root.Q_Ub[-1,:,:],
'Q_Uc' : h5.root.Q_Uc[-1,:,:],
}
model = hparams['model']
model.set_model_params(mparams)
n_hid, n_vis = mparams['P_W'].shape
assert n_vis == model.n_vis
assert n_hid == model.n_hid
clf()
figsize(*figaspect(0.3))
title("est. Log-Likelihood"); xlabel("Epoch")
for var, label in ( (Lp_10, 'L_{10}'), (Lp_25, 'L_{25}'), (Lp_100, 'L_{100}'),):
plot(var[:], label="$%s$ = %5.2f"%(label, var[-1]));
legend(loc='lower right')
#s = 0
#print Lp_10[s], Lp_25[s], Lp_100[s]
Out[58]:
In [4]:
display_fields(model.P_W.get_value())
In [9]:
# Compile some model functions
n_samples = T.iscalar('n_samples')
H, Ph = model.f_ph_sample(n_samples)
do_ph_sample = theano.function([n_samples], [H, Ph], name='do_ph_sample')
H = T.fmatrix('H')
X, Px = model.f_p_sample(H)
Hq,Pq = model.f_q_sample(X)
idx = T.argsort(Px)[::-1]
Hq = Hq[idx]
X = X[idx]
do_p_sample = theano.function([H], [X, H], name='do_p_sample')
n_samples = T.iscalar('n_samples')
X = T.fmatrix('X')
lP, lQ, lPx, lQx, H, w = model.f_loglikelihood(X, n_samples=n_samples)
do_loglikelihood = theano.function(
[X, n_samples],
[lP, lQ, lPx, lQx, H, w],
name='loglikelihood')
In [38]:
#H, Ph = do_ph_sample(100)
for i in xrange(100):
X, H = do_p_sample(H)
display_fields(X[:,:], D=28)
In [37]:
display_fields(P_vis)
In [45]:
mnist = MNIST(which_set='test')
test_X = mnist.X
In [46]:
def estimate_LL(X, n_samples=100):
batch_size = 100
N, _ = X.shape
assert N % batch_size == 0
lP = np.empty((N, n_samples), dtype=np.float32)
lPx = np.empty((N), dtype=np.float32)
for i in xrange(N//batch_size):
first = i*batch_size
last = first + batch_size
_lP, _, _lPx, _, _, _ = do_loglikelihood(X[first:last], n_samples)
lP[first:last] = _lP
lPx[first:last] = _lPx[:,0]
return lP, lPx
In [48]:
lP, _, lPx, _, H, w = do_loglikelihood(test_X[:100], 100)
In [47]:
display_fields(test_X)
In [49]:
idx = np.argsort(lP[0])[::-1]
lP = lP[idx]
H[0,:,:] = H[0,idx,:]
In [54]:
W = mparams['P_W']
b = mparams['P_b']
In [55]:
imshow(test_X[0].reshape((28,28)), interpolation='nearest')
Out[55]:
In [57]:
h = H[0,0,:]
X = 1./(1+np.exp(-np.dot(W.T, h) - b))
imshow(X.reshape((28,28)), interpolation='nearest')
Out[57]:
In [103]:
n_samples = T.iscalar('n_samples')
model.f_p_sample(n_samples)
Out[103]:
In [116]:
n_samples = [1, 5, 25, 100, 500]
LL = []
for spl in n_samples:
print "Estimating LL with %d samples" % spl
_, lPx = estimate_LL(test_X[:], spl)
LL.append(lPx.mean())
title("Loglikelihood on MNIST testset")
semilogx(n_samples, LL, "o-")
ylabel("$\cal L$")
xlabel("# samples")
print "LL", LL[-1]
In [126]:
Motre detailed analyss
In [113]:
lP, lPx = estimate_LL(test_X[:, :], 50)
In [126]:
print "Mean: ", np.mean(lPx)
print "Median: ", np.median(lPx)
_ = hist(lPx)
In [121]:
idx = np.argsort(lPx)
X_ = test_X[idx]
lPx_ = lPx[idx]
print "worst p(x)"
display_fields(X_[0:])
In [122]:
print "best p(x)"
display_fields(X_[-24:])
In [ ]: