Experiments with Toy (synthetic) dataset

Generate data

$E \sim N(0, \sigma_e^2 I)$

for each $k$

$R_k \sim MN(0, \sigma_r^2 I, I)$

$X_k \sim N(ER_kE^\top, \sigma_x^2)$

and fit the models with this dataset.


In [11]:
import numpy as np
import logging
import itertools
from scipy.sparse import csr_matrix
import rescal
from almc.bayesian_rescal import BayesianRescal

%matplotlib inline  
logger = logging.getLogger()
logger.setLevel(logging.INFO)

max_iter = 20

n_entity = 10
n_dim = 5
n_relation = 20

var_e = 1.
var_r = 1.
var_x = 1.
e_mean = np.zeros(n_dim)
r_mean = np.zeros(n_dim**2)
E = np.random.multivariate_normal(e_mean, np.identity(n_dim) * var_e, size = n_entity)
R = np.zeros([n_relation, n_dim, n_dim])
for k in range(n_relation):
    R[k] = np.random.multivariate_normal(r_mean, np.identity(n_dim**2)*var_r).reshape(n_dim,n_dim)
X = np.zeros([n_relation, n_entity, n_entity])
for k in range(n_relation):
    ERET = np.dot(np.dot(E, R[k]), E.T)
    for i,j in itertools.product(range(n_entity), repeat=2):
        X[k,i,j] = np.random.normal(ERET[i,j], var_x)

In [12]:
import itertools
trainX = np.zeros_like(X)
p = 1
for k in range(n_relation):
    for i,j in itertools.product(range(n_entity),repeat=2):
        if X[k, i, j] and np.random.binomial(1, p):
            trainX[k, i, j] = X[k, i, j]

In [13]:
model = BayesianRescal(n_dim, var_e=var_e, var_x=var_x, var_r=var_r)
model.fit(trainX, max_iter=max_iter)


INFO:brescal:[INIT] LL: -60480.755 | fit: -0.42100
INFO:brescal:[  0] LL: -19991.532 | fit: 0.61257 |  sec: 0.020
INFO:brescal:[  1] LL: -5961.656 | fit: 0.95448 |  sec: 0.018
INFO:brescal:[  2] LL: -5207.340 | fit: 0.96671 |  sec: 0.021
INFO:brescal:[  3] LL: -4836.556 | fit: 0.97027 |  sec: 0.022
INFO:brescal:[  4] LL: -4603.180 | fit: 0.97219 |  sec: 0.020
INFO:brescal:[  5] LL: -4440.849 | fit: 0.97322 |  sec: 0.020
INFO:brescal:[  6] LL: -4345.156 | fit: 0.97387 |  sec: 0.020
INFO:brescal:[  7] LL: -4261.078 | fit: 0.97432 |  sec: 0.021
INFO:brescal:[  8] LL: -4246.298 | fit: 0.97422 |  sec: 0.021
INFO:brescal:[  9] LL: -4190.940 | fit: 0.97492 |  sec: 0.026
INFO:brescal:[ 10] LL: -4178.886 | fit: 0.97443 |  sec: 0.022
INFO:brescal:[ 11] LL: -4167.557 | fit: 0.97387 |  sec: 0.020
INFO:brescal:[ 12] LL: -4090.523 | fit: 0.97534 |  sec: 0.019
INFO:brescal:[ 13] LL: -4072.417 | fit: 0.97449 |  sec: 0.020
INFO:brescal:[ 14] LL: -4018.654 | fit: 0.97481 |  sec: 0.020
INFO:brescal:[ 15] LL: -3977.947 | fit: 0.97490 |  sec: 0.019
INFO:brescal:[ 16] LL: -3937.127 | fit: 0.97547 |  sec: 0.020
INFO:brescal:[ 17] LL: -3912.645 | fit: 0.97568 |  sec: 0.020
INFO:brescal:[ 18] LL: -3909.708 | fit: 0.97515 |  sec: 0.020
INFO:brescal:[ 19] LL: -3870.625 | fit: 0.97586 |  sec: 0.021

In [14]:
csr_X = list()
for k in range(n_relation):
    csr_X.append(csr_matrix(trainX[k]))
    
E, R, f, itr, exectimes = rescal.rescal_als(csr_X, n_dim)


INFO:RESCAL:[  0] fit: 0.98179 | delta: 9.8e-01 | secs: 0.00663
INFO:RESCAL:[  1] fit: 0.98192 | delta: 1.3e-04 | secs: 0.00833
INFO:RESCAL:[  2] fit: 0.98192 | delta: 2.9e-07 | secs: 0.00981

In [15]:
_X = model._reconstruct()

print("BayesRESCAL:Element-wise squared error: %.3f" %(np.sum((X-_X)**2)))

_X = np.zeros_like(X)
for k in range(n_relation):
    _X[k] = np.dot(np.dot(E, R[k]), E.T)
    
print("RESCAL:Element-wise squared error: %.3f" %(np.sum((X-_X)**2)))


BayesRESCAL:Element-wise squared error: 1971.791
RESCAL:Element-wise squared error: 1477.210

Split data into training/test data

... but both models consider the input tensor as a full tensor.


In [16]:
import itertools
trainX = np.zeros_like(X)
p = 0.5 # proportion of training data points
for k in range(n_relation):
    for i,j in itertools.product(range(n_entity),repeat=2):
        if X[k, i, j] and np.random.binomial(1, p):
            trainX[k, i, j] = X[k, i, j]
            
model = BayesianRescal(n_dim, var_e=var_e, var_x=var_x, var_r=var_r)
model.fit(trainX, max_iter=max_iter)


INFO:brescal:[INIT] LL: -36683.364 | fit: -0.61710
INFO:brescal:[  0] LL: -16153.299 | fit: 0.37381 |  sec: 0.024
INFO:brescal:[  1] LL: -12296.431 | fit: 0.56791 |  sec: 0.019
INFO:brescal:[  2] LL: -10921.524 | fit: 0.63153 |  sec: 0.020
INFO:brescal:[  3] LL: -10578.255 | fit: 0.64262 |  sec: 0.020
INFO:brescal:[  4] LL: -10401.813 | fit: 0.64773 |  sec: 0.020
INFO:brescal:[  5] LL: -10276.329 | fit: 0.64935 |  sec: 0.020
INFO:brescal:[  6] LL: -10204.490 | fit: 0.65081 |  sec: 0.020
INFO:brescal:[  7] LL: -10128.645 | fit: 0.65272 |  sec: 0.020
INFO:brescal:[  8] LL: -10088.756 | fit: 0.65335 |  sec: 0.019
INFO:brescal:[  9] LL: -10094.159 | fit: 0.65310 |  sec: 0.020
INFO:brescal:[ 10] LL: -10081.900 | fit: 0.65400 |  sec: 0.019
INFO:brescal:[ 11] LL: -10077.738 | fit: 0.65359 |  sec: 0.019
INFO:brescal:[ 12] LL: -10086.612 | fit: 0.65247 |  sec: 0.019
INFO:brescal:[ 13] LL: -10062.461 | fit: 0.65352 |  sec: 0.019
INFO:brescal:[ 14] LL: -10071.100 | fit: 0.65297 |  sec: 0.019
INFO:brescal:[ 15] LL: -10048.566 | fit: 0.65432 |  sec: 0.020
INFO:brescal:[ 16] LL: -10016.412 | fit: 0.65581 |  sec: 0.019
INFO:brescal:[ 17] LL: -9989.560 | fit: 0.65619 |  sec: 0.020
INFO:brescal:[ 18] LL: -10022.224 | fit: 0.65450 |  sec: 0.020
INFO:brescal:[ 19] LL: -9970.158 | fit: 0.65687 |  sec: 0.020

In [17]:
csr_X = list()
for k in range(n_relation):
    csr_X.append(csr_matrix(trainX[k]))
    
E, R, f, itr, exectimes = rescal.rescal_als(csr_X, n_dim)


INFO:RESCAL:[  0] fit: 0.63314 | delta: 6.3e-01 | secs: 0.00675
INFO:RESCAL:[  1] fit: 0.66118 | delta: 2.8e-02 | secs: 0.00777
INFO:RESCAL:[  2] fit: 0.66740 | delta: 6.2e-03 | secs: 0.00953
INFO:RESCAL:[  3] fit: 0.66932 | delta: 1.9e-03 | secs: 0.00762
INFO:RESCAL:[  4] fit: 0.67027 | delta: 9.5e-04 | secs: 0.00849
INFO:RESCAL:[  5] fit: 0.67091 | delta: 6.5e-04 | secs: 0.00774
INFO:RESCAL:[  6] fit: 0.67143 | delta: 5.1e-04 | secs: 0.00872
INFO:RESCAL:[  7] fit: 0.67186 | delta: 4.4e-04 | secs: 0.00841
INFO:RESCAL:[  8] fit: 0.67225 | delta: 3.9e-04 | secs: 0.00890
INFO:RESCAL:[  9] fit: 0.67260 | delta: 3.5e-04 | secs: 0.00758
INFO:RESCAL:[ 10] fit: 0.67291 | delta: 3.1e-04 | secs: 0.00843
INFO:RESCAL:[ 11] fit: 0.67319 | delta: 2.8e-04 | secs: 0.00679
INFO:RESCAL:[ 12] fit: 0.67345 | delta: 2.6e-04 | secs: 0.00770
INFO:RESCAL:[ 13] fit: 0.67368 | delta: 2.3e-04 | secs: 0.00753
INFO:RESCAL:[ 14] fit: 0.67389 | delta: 2.1e-04 | secs: 0.00828
INFO:RESCAL:[ 15] fit: 0.67407 | delta: 1.8e-04 | secs: 0.00785
INFO:RESCAL:[ 16] fit: 0.67424 | delta: 1.6e-04 | secs: 0.00755
INFO:RESCAL:[ 17] fit: 0.67438 | delta: 1.4e-04 | secs: 0.00792
INFO:RESCAL:[ 18] fit: 0.67451 | delta: 1.3e-04 | secs: 0.00731
INFO:RESCAL:[ 19] fit: 0.67461 | delta: 1.1e-04 | secs: 0.00757
INFO:RESCAL:[ 20] fit: 0.67471 | delta: 9.5e-05 | secs: 0.00817

In [18]:
_bX = model._reconstruct()

print("BayesRESCAL:Element-wise squared error on training data: %.3f" %(np.sum((trainX-_bX)**2)))
print("BayesRESCAL:Element-wise squared error on test data: %.3f\n" %(np.sum((X-_bX)[trainX==0]**2)))

_X = np.zeros_like(X)
for k in range(n_relation):
    _X[k] = np.dot(np.dot(E, R[k]), E.T)
    
print("RESCAL:Element-wise squared error on training data: %.3f" %(np.sum((trainX-_X)**2)))
print("RESCAL:Element-wise squared error on test data: %.3f" %(np.sum((X-_X)[trainX==0]**2)))


BayesRESCAL:Element-wise squared error on training data: 14533.638
BayesRESCAL:Element-wise squared error on test data: 23437.408

RESCAL:Element-wise squared error on training data: 13778.150
RESCAL:Element-wise squared error on test data: 20795.047

Fit

Fit $= 1 - A / B$

$A = \sum_{i,k,j}(x_{i,k,j} - \hat{x}_{i,k,j})^2 \quad\quad$ where $\hat{x}$ is a predicted value of triple $(i,k,j)$. This is the sum of squared error.

$B = \sum_{i,k,j}(x_{i,k,j}) \quad\quad\quad\quad$ = Sum of squared datapoints.

$A$ is the objective function of RESCAL


In [19]:
A = np.sum((trainX-_X)**2)
B = np.sum(trainX**2)
fit = 1.-A/B
print(fit)


0.674710002298

Control variance of observed/unobserved data

Placing different variances (var_x) on observed/unobserved points will improve the factorisation performance. (Implicit feedback)

Variance of observed elements (obs_var) = 1

Variance of unboserved elements (unobs_var) = 100 (=almost do not care about unobserved data points)


In [20]:
model = BayesianRescal(n_dim, var_e=var_e, var_x=var_x, var_r=var_r, controlled_var=True, obs_var=1., unobs_var=100.)
model.fit(trainX, max_iter=20)
_cX = model._reconstruct()

print("BayesRESCAL:Element-wise squared error on training data: %.3f" %(np.sum((trainX[trainX!=0]-_bX[trainX!=0])**2)))
print("BayesRESCAL:Element-wise squared error on test data: %.3f\n" %(np.sum((X-_bX)[trainX==0]**2)))

print("RESCAL:Element-wise squared error on training data: %.3f" %(np.sum((trainX[trainX!=0]-_X[trainX!=0])**2)))
print("RESCAL:Element-wise squared error on test data: %.3f\n" %(np.sum((X-_X)[trainX==0]**2)))

print("CV_BayesRESCAL:Element-wise squared error on training data: %.3f" %(np.sum((trainX[trainX!=0]-_cX[trainX!=0])**2)))
print("CV_BayesRESCAL:Element-wise squared error on test data: %.3f" %(np.sum((X-_cX)[trainX==0]**2)))


INFO:brescal:[INIT] LL: -29928.359 | fit: -0.29810
INFO:brescal:[  0] LL: -28940.870 | fit: -0.21057 |  sec: 0.027
INFO:brescal:[  1] LL: -23098.370 | fit: 0.07060 |  sec: 0.026
INFO:brescal:[  2] LL: -19467.511 | fit: 0.23781 |  sec: 0.027
INFO:brescal:[  3] LL: -19496.439 | fit: 0.23435 |  sec: 0.025
INFO:brescal:[  4] LL: -19326.661 | fit: 0.23688 |  sec: 0.027
INFO:brescal:[  5] LL: -19298.488 | fit: 0.23555 |  sec: 0.026
INFO:brescal:[  6] LL: -19664.847 | fit: 0.21585 |  sec: 0.026
INFO:brescal:[  7] LL: -19634.413 | fit: 0.21517 |  sec: 0.027
INFO:brescal:[  8] LL: -19771.240 | fit: 0.20689 |  sec: 0.026
INFO:brescal:[  9] LL: -20290.738 | fit: 0.18100 |  sec: 0.027
INFO:brescal:[ 10] LL: -20297.768 | fit: 0.17999 |  sec: 0.053
INFO:brescal:[ 11] LL: -20393.801 | fit: 0.17456 |  sec: 0.033
INFO:brescal:[ 12] LL: -20108.642 | fit: 0.18718 |  sec: 0.034
INFO:brescal:[ 13] LL: -20463.116 | fit: 0.17008 |  sec: 0.028
INFO:brescal:[ 14] LL: -20409.298 | fit: 0.17240 |  sec: 0.036
INFO:brescal:[ 15] LL: -20554.694 | fit: 0.16511 |  sec: 0.049
INFO:brescal:[ 16] LL: -20280.350 | fit: 0.17738 |  sec: 0.027
INFO:brescal:[ 17] LL: -20147.844 | fit: 0.18360 |  sec: 0.029
INFO:brescal:[ 18] LL: -19984.571 | fit: 0.19059 |  sec: 0.027
INFO:brescal:[ 19] LL: -20031.436 | fit: 0.18736 |  sec: 0.051
BayesRESCAL:Element-wise squared error on training data: 9341.559
BayesRESCAL:Element-wise squared error on test data: 23437.408

RESCAL:Element-wise squared error on training data: 7744.750
RESCAL:Element-wise squared error on test data: 20795.047

CV_BayesRESCAL:Element-wise squared error on training data: 995.098
CV_BayesRESCAL:Element-wise squared error on test data: 3095.573

Result: Controlled variance approach outperforms other approaches