Experiments with Similarity Encoders (SimEc)

To show that SimEc can predict pairwise relations between data points and learn similarity preserving embeddings by mapping feature vectors into an embedding space where a given target similarity matrix can be approximated by the scalar product of the embedding vectors.



In [1]:

    
from __future__ import unicode_literals, division, print_function, absolute_import
from builtins import range
import numpy as np
np.random.seed(28)
import matplotlib.pyplot as plt
import tensorflow as tf
tf.set_random_seed(28)
import keras

from scipy.spatial.distance import pdist, squareform
from sklearn.decomposition import PCA, KernelPCA
from sklearn.linear_model import Ridge
from sklearn.model_selection import GridSearchCV
from sklearn.datasets import fetch_mldata
from sklearn.preprocessing import StandardScaler
from sklearn.metrics.pairwise import rbf_kernel

from simec import SimilarityEncoder
from utils import center_K, check_similarity_match
from utils_plotting import get_colors, plot_mnist, plot_mnist2

%matplotlib inline
%load_ext autoreload
%autoreload 2
# set this to True if you want to save the figures from the paper
savefigs = True









    



Using TensorFlow backend.



In [2]:

    
# load digits
mnist = fetch_mldata('MNIST original', data_home='data')
X = mnist.data/255.  # normalize to 0-1
y = np.array(mnist.target, dtype=int)
# subsample 10000 random data points
np.random.seed(42)
n_samples = 10000
n_test = 2000
rnd_idx = np.random.permutation(X.shape[0])[:n_samples]
X_test, y_test = X[rnd_idx[:n_test],:], y[rnd_idx[:n_test]]
X, y = X[rnd_idx[n_test:],:], y[rnd_idx[n_test:]]
# scale
ss = StandardScaler(with_std=False)
X = ss.fit_transform(X)
X_test = ss.transform(X_test)
n_train, n_features = X.shape

Embedding MNIST based on class based similarities



In [3]:

    
# compute similarity matrix based on class labels
Y = np.tile(y, (len(y), 1))
S = center_K(np.array(Y==Y.T, dtype=int))
Y = np.tile(y_test, (len(y_test), 1))
S_test = center_K(np.array(Y==Y.T, dtype=int))



In [4]:

    
# compute the eigendecomposition of S as the perfect solution
D, V = np.linalg.eig(S)
# embedding based on largest EV
D1, V1 = D[np.argsort(D)[::-1]], V[:,np.argsort(D)[::-1]]
X_embed = np.dot(V1.real, np.diag(np.sqrt(np.abs(D1.real))))
# check how many relevant dimensions there are
plt.figure();
plt.plot(list(range(1, S.shape[0]+1)), D1.real, '-o', markersize=3);
plt.plot([1, S.shape[0]],[0,0], 'k--', linewidth=0.5);
plt.xlim(1, 100);
plt.title('Eigenvalue spectrum of S (based on class labels)');
# the embedding itself is not so impressive...
plot_mnist(X_embed[:,:2], y, title='MNIST (class based) - largest 2 EV')



In [5]:

    
n_targets = 2000
# get good alpha for RR model
m = Ridge()
rrm = GridSearchCV(m, {'alpha': [0.000001, 0.00001, 0.0001, 0.001, 0.01, 0.1, 0.25, 0.5, 0.75, 1., 2.5, 5., 7.5, 10., 25., 50., 75., 100., 250., 500., 750., 1000.]})
rrm.fit(X, X_embed[:,:8])
alpha = rrm.best_params_["alpha"]
print("Ridge Regression with alpha: %r" % alpha)
mse_ev, mse_rr, mse_rr_test = [], [], []
mse_simec, mse_simec_test = [], []
mse_simec_hl, mse_simec_hl_test = [], []
mse_simec_5hl, mse_simec_5hl_test = [], []
e_dims = [2, 3, 4, 5, 6, 7, 8, 9, 10, 15]
for e_dim in e_dims:
    print(e_dim)
    # eigenvalue based embedding
    mse = check_similarity_match(X_embed[:,:e_dim], S)[0]
    mse_ev.append(mse)
    # train a linear ridge regression model to learn the mapping from X to Y
    model = Ridge(alpha=alpha)
    model.fit(X, X_embed[:,:e_dim])
    X_embed_r = model.predict(X)
    X_embed_test_r = model.predict(X_test)
    mse = check_similarity_match(X_embed_r, S)[0]
    mse_rr.append(mse)
    mse = check_similarity_match(X_embed_test_r, S_test)[0]
    mse_rr_test.append(mse)
    # simec - linear
    simec = SimilarityEncoder(X.shape[1], e_dim, n_targets, s_ll_reg=0.5, S_ll=S[:n_targets,:n_targets],
                              orth_reg=0.005 if e_dim > 7 else 0., l2_reg_emb=0.00001, 
                              l2_reg_out=0.0000001, opt=keras.optimizers.Adamax(lr=0.001))
    simec.fit(X, S[:,:n_targets])
    X_embeds = simec.transform(X)
    X_embed_tests = simec.transform(X_test)
    mse = check_similarity_match(X_embeds, S)[0]
    mse_simec.append(mse)
    mse_t = check_similarity_match(X_embed_tests, S_test)[0]
    mse_simec_test.append(mse_t)
    # simec - 2hl
    o = 0.01 if e_dim > 7 else 0.
    if e_dim == 15:
        o = 0.1
    simec = SimilarityEncoder(X.shape[1], e_dim, n_targets, hidden_layers=[(25, 'tanh'), (25, 'tanh')],
                              s_ll_reg=0.5, S_ll=S[:n_targets,:n_targets], orth_reg=o, 
                              l2_reg=0., l2_reg_emb=0.00001, l2_reg_out=0.0000001, opt=keras.optimizers.Adamax(lr=0.005))
    simec.fit(X, S[:,:n_targets])
    X_embeds = simec.transform(X)
    X_embed_tests = simec.transform(X_test)
    mse = check_similarity_match(X_embeds, S)[0]
    mse_simec_hl.append(mse)
    mse_t = check_similarity_match(X_embed_tests, S_test)[0]
    mse_simec_hl_test.append(mse_t)
    # simec - 5hl
    simec = SimilarityEncoder(X.shape[1], e_dim, n_targets, hidden_layers=[(25, 'tanh'), (25, 'tanh'), (25, 'tanh'), (25, 'tanh'), (25, 'tanh')],
                              s_ll_reg=0.5, S_ll=S[:n_targets,:n_targets], orth_reg=0.01 if e_dim > 7 else 0., 
                              l2_reg=0., l2_reg_emb=0.00001, l2_reg_out=0.0000001, opt=keras.optimizers.Adamax(lr=0.005))
    simec.fit(X, S[:,:n_targets])
    X_embeds = simec.transform(X)
    X_embed_tests = simec.transform(X_test)
    mse = check_similarity_match(X_embeds, S)[0]
    mse_simec_5hl.append(mse)
    mse_t = check_similarity_match(X_embed_tests, S_test)[0]
    mse_simec_5hl_test.append(mse_t)
    print("mse ev: %f; mse rr: %f (%f); mse simec (0hl): %f (%f); mse simec (2hl): %f (%f); mse simec (5hl): %f (%f)" % (mse_ev[-1], mse_rr[-1], mse_rr_test[-1], mse_simec[-1], mse_simec_test[-1], mse_simec_hl[-1], mse_simec_hl_test[-1], mse, mse_t))
keras.backend.clear_session()
print("e_dims=", e_dims)
print("mse_ev=", mse_ev)
print("mse_rr=", mse_rr)
print("mse_rr_test=", mse_rr_test)
print("mse_simec=", mse_simec)
print("mse_simec_test=", mse_simec_test)
print("mse_simec_hl=", mse_simec_hl)
print("mse_simec_hl_test=", mse_simec_hl_test)
print("mse_simec_5hl=", mse_simec_5hl)
print("mse_simec_5hl_test=", mse_simec_5hl_test)
colors = get_colors(15)
plt.figure();
plt.plot(e_dims, mse_ev, '-o', markersize=3, c=colors[14], label='Eigendecomp.');
plt.plot(e_dims, mse_rr, '-o', markersize=3, c=colors[12], label='ED + Regression');
plt.plot(e_dims, mse_rr_test, '--o', markersize=3, c=colors[12], label='ED + Reg. (test)');
plt.plot(e_dims, mse_simec, '-o', markersize=3, c=colors[9], label='SimEc 0hl');
plt.plot(e_dims, mse_simec_test, '--o', markersize=3, c=colors[9], label='SimEc 0hl (test)');
plt.plot(e_dims, mse_simec_hl, '-o', markersize=3, c=colors[8], label='SimEc 2hl');
plt.plot(e_dims, mse_simec_hl_test, '--o', markersize=3, c=colors[8], label='SimEc 2hl (test)');
plt.plot(e_dims, mse_simec_5hl, '-o', markersize=3, c=colors[4], label='SimEc 5hl');
plt.plot(e_dims, mse_simec_5hl_test, '--o', markersize=3, c=colors[4], label='SimEc 5hl (test)');
plt.legend(loc=3);
plt.title('MNIST (class based similarities)');
plt.plot([0, e_dims[-1]], [0,0], 'k--', linewidth=0.5);
plt.xticks(e_dims, e_dims);
plt.xlabel('Number of Embedding Dimensions ($d$)')
plt.ylabel('Mean Squared Error of $\hat{S}$')
plt.xlim([0,11])
if savefigs: plt.savefig('fig_class_mse_edim.pdf', dpi=300, bbox_inches="tight")









    



Ridge Regression with alpha: 75.0
2
Epoch 1/25
8000/8000 [==============================] - 2s 219us/step - loss: 0.1282
Epoch 2/25
8000/8000 [==============================] - 1s 140us/step - loss: 0.1199
Epoch 3/25
8000/8000 [==============================] - 1s 137us/step - loss: 0.1188
Epoch 4/25
8000/8000 [==============================] - 1s 139us/step - loss: 0.1182
Epoch 5/25
8000/8000 [==============================] - 1s 141us/step - loss: 0.1177
Epoch 6/25
8000/8000 [==============================] - 1s 143us/step - loss: 0.1173
Epoch 7/25
8000/8000 [==============================] - 1s 134us/step - loss: 0.1168
Epoch 8/25
8000/8000 [==============================] - 1s 139us/step - loss: 0.1163
Epoch 9/25
8000/8000 [==============================] - 1s 137us/step - loss: 0.1157
Epoch 10/25
8000/8000 [==============================] - 1s 138us/step - loss: 0.1150
Epoch 11/25
8000/8000 [==============================] - 1s 134us/step - loss: 0.1140
Epoch 12/25
8000/8000 [==============================] - 1s 136us/step - loss: 0.1128
Epoch 13/25
8000/8000 [==============================] - 1s 137us/step - loss: 0.1113
Epoch 14/25
8000/8000 [==============================] - 1s 139us/step - loss: 0.1096
Epoch 15/25
8000/8000 [==============================] - 1s 140us/step - loss: 0.1081
Epoch 16/25
8000/8000 [==============================] - 1s 144us/step - loss: 0.1070
Epoch 17/25
8000/8000 [==============================] - 1s 140us/step - loss: 0.1064
Epoch 18/25
8000/8000 [==============================] - 1s 138us/step - loss: 0.1061
Epoch 19/25
8000/8000 [==============================] - 1s 138us/step - loss: 0.1059
Epoch 20/25
8000/8000 [==============================] - 1s 138us/step - loss: 0.1059
Epoch 21/25
8000/8000 [==============================] - 1s 140us/step - loss: 0.1058
Epoch 22/25
8000/8000 [==============================] - 1s 136us/step - loss: 0.1057
Epoch 23/25
8000/8000 [==============================] - 1s 137us/step - loss: 0.1057
Epoch 24/25
8000/8000 [==============================] - 1s 141us/step - loss: 0.1056
Epoch 25/25
8000/8000 [==============================] - 1s 138us/step - loss: 0.1056
Epoch 1/25
8000/8000 [==============================] - 2s 206us/step - loss: 0.1196
Epoch 2/25
8000/8000 [==============================] - 1s 168us/step - loss: 0.1134
Epoch 3/25
8000/8000 [==============================] - 1s 169us/step - loss: 0.1091
Epoch 4/25
8000/8000 [==============================] - 1s 168us/step - loss: 0.1038
Epoch 5/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.1017
Epoch 6/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.1012
Epoch 7/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.1010
Epoch 8/25
8000/8000 [==============================] - 1s 160us/step - loss: 0.1008
Epoch 9/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.1007
Epoch 10/25
8000/8000 [==============================] - 1s 168us/step - loss: 0.1006
Epoch 11/25
8000/8000 [==============================] - 1s 170us/step - loss: 0.1004
Epoch 12/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.1003
Epoch 13/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.1003
Epoch 14/25
8000/8000 [==============================] - 1s 170us/step - loss: 0.1002
Epoch 15/25
8000/8000 [==============================] - 1s 160us/step - loss: 0.1001
Epoch 16/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.1001
Epoch 17/25
8000/8000 [==============================] - 1s 172us/step - loss: 0.1000
Epoch 18/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.1000
Epoch 19/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0999
Epoch 20/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0999
Epoch 21/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0998
Epoch 22/25
8000/8000 [==============================] - 1s 167us/step - loss: 0.0998
Epoch 23/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0998
Epoch 24/25
8000/8000 [==============================] - 1s 168us/step - loss: 0.0998
Epoch 25/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0997
Epoch 1/25
8000/8000 [==============================] - 2s 241us/step - loss: 0.1183
Epoch 2/25
8000/8000 [==============================] - 2s 200us/step - loss: 0.1125
Epoch 3/25
8000/8000 [==============================] - 2s 204us/step - loss: 0.1084
Epoch 4/25
8000/8000 [==============================] - 2s 200us/step - loss: 0.1029
Epoch 5/25
8000/8000 [==============================] - 2s 201us/step - loss: 0.1009
Epoch 6/25
8000/8000 [==============================] - 2s 201us/step - loss: 0.1005
Epoch 7/25
8000/8000 [==============================] - 2s 202us/step - loss: 0.1002
Epoch 8/25
8000/8000 [==============================] - 2s 204us/step - loss: 0.1001
Epoch 9/25
8000/8000 [==============================] - 2s 200us/step - loss: 0.0999
Epoch 10/25
8000/8000 [==============================] - 2s 200us/step - loss: 0.0998
Epoch 11/25
8000/8000 [==============================] - 2s 200us/step - loss: 0.0998
Epoch 12/25
8000/8000 [==============================] - 2s 204us/step - loss: 0.0997
Epoch 13/25
8000/8000 [==============================] - 2s 201us/step - loss: 0.0996
Epoch 14/25
8000/8000 [==============================] - 2s 200us/step - loss: 0.0996
Epoch 15/25
8000/8000 [==============================] - 2s 201us/step - loss: 0.0995
Epoch 16/25
8000/8000 [==============================] - 2s 206us/step - loss: 0.0995
Epoch 17/25
8000/8000 [==============================] - 2s 196us/step - loss: 0.0995
Epoch 18/25
8000/8000 [==============================] - 2s 196us/step - loss: 0.0995
Epoch 19/25
8000/8000 [==============================] - 2s 203us/step - loss: 0.0995
Epoch 20/25
8000/8000 [==============================] - 2s 204us/step - loss: 0.0995
Epoch 21/25
8000/8000 [==============================] - 2s 196us/step - loss: 0.0995
Epoch 22/25
8000/8000 [==============================] - 2s 201us/step - loss: 0.0994
Epoch 23/25
8000/8000 [==============================] - 2s 198us/step - loss: 0.0995
Epoch 24/25
8000/8000 [==============================] - 2s 198us/step - loss: 0.0994
Epoch 25/25
8000/8000 [==============================] - 2s 196us/step - loss: 0.0994
mse ev: 0.066420; mse rr: 0.077786 (0.079355); mse simec (0hl): 0.076568 (0.078448); mse simec (2hl): 0.067427 (0.072142); mse simec (5hl): 0.067052 (0.071176)
3
Epoch 1/25
8000/8000 [==============================] - 1s 177us/step - loss: 0.1247
Epoch 2/25
8000/8000 [==============================] - 1s 138us/step - loss: 0.1128
Epoch 3/25
8000/8000 [==============================] - 1s 139us/step - loss: 0.1107
Epoch 4/25
8000/8000 [==============================] - 1s 141us/step - loss: 0.1097
Epoch 5/25
8000/8000 [==============================] - 1s 143us/step - loss: 0.1089
Epoch 6/25
8000/8000 [==============================] - 1s 137us/step - loss: 0.1082
Epoch 7/25
8000/8000 [==============================] - 1s 141us/step - loss: 0.1073
Epoch 8/25
8000/8000 [==============================] - 1s 141us/step - loss: 0.1062
Epoch 9/25
8000/8000 [==============================] - 1s 141us/step - loss: 0.1048
Epoch 10/25
8000/8000 [==============================] - 1s 136us/step - loss: 0.1027
Epoch 11/25
8000/8000 [==============================] - 1s 141us/step - loss: 0.1003
Epoch 12/25
8000/8000 [==============================] - 1s 138us/step - loss: 0.0977
Epoch 13/25
8000/8000 [==============================] - 1s 133us/step - loss: 0.0955
Epoch 14/25
8000/8000 [==============================] - 1s 134us/step - loss: 0.0940
Epoch 15/25
8000/8000 [==============================] - 1s 140us/step - loss: 0.0931
Epoch 16/25
8000/8000 [==============================] - 1s 138us/step - loss: 0.0926
Epoch 17/25
8000/8000 [==============================] - 1s 138us/step - loss: 0.0924
Epoch 18/25
8000/8000 [==============================] - 1s 136us/step - loss: 0.0923
Epoch 19/25
8000/8000 [==============================] - 1s 139us/step - loss: 0.0922
Epoch 20/25
8000/8000 [==============================] - 1s 139us/step - loss: 0.0922
Epoch 21/25
8000/8000 [==============================] - 1s 139us/step - loss: 0.0921
Epoch 22/25
8000/8000 [==============================] - 1s 137us/step - loss: 0.0920
Epoch 23/25
8000/8000 [==============================] - 1s 135us/step - loss: 0.0920
Epoch 24/25
8000/8000 [==============================] - 1s 140us/step - loss: 0.0920
Epoch 25/25
8000/8000 [==============================] - 1s 137us/step - loss: 0.0919
Epoch 1/25
8000/8000 [==============================] - 2s 211us/step - loss: 0.1133
Epoch 2/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.1024
Epoch 3/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0949
Epoch 4/25
8000/8000 [==============================] - 1s 169us/step - loss: 0.0880
Epoch 5/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0856
Epoch 6/25
8000/8000 [==============================] - 1s 169us/step - loss: 0.0850
Epoch 7/25
8000/8000 [==============================] - 1s 170us/step - loss: 0.0848
Epoch 8/25
8000/8000 [==============================] - 1s 171us/step - loss: 0.0845
Epoch 9/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0843
Epoch 10/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0842
Epoch 11/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0841
Epoch 12/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0839
Epoch 13/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0838
Epoch 14/25
8000/8000 [==============================] - 1s 169us/step - loss: 0.0837
Epoch 15/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0836
Epoch 16/25
8000/8000 [==============================] - 1s 168us/step - loss: 0.0836
Epoch 17/25
8000/8000 [==============================] - 1s 171us/step - loss: 0.0835
Epoch 18/25
8000/8000 [==============================] - 1s 171us/step - loss: 0.0834
Epoch 19/25
8000/8000 [==============================] - 1s 173us/step - loss: 0.0834
Epoch 20/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0833
Epoch 21/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0832
Epoch 22/25
8000/8000 [==============================] - 1s 167us/step - loss: 0.0832
Epoch 23/25
8000/8000 [==============================] - 1s 168us/step - loss: 0.0831
Epoch 24/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0831
Epoch 25/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0830
Epoch 1/25
8000/8000 [==============================] - 2s 251us/step - loss: 0.1109
Epoch 2/25
8000/8000 [==============================] - 2s 203us/step - loss: 0.1013
Epoch 3/25
8000/8000 [==============================] - 2s 206us/step - loss: 0.0934
Epoch 4/25
8000/8000 [==============================] - 2s 200us/step - loss: 0.0861
Epoch 5/25
8000/8000 [==============================] - 2s 196us/step - loss: 0.0843
Epoch 6/25
8000/8000 [==============================] - 2s 200us/step - loss: 0.0839
Epoch 7/25
8000/8000 [==============================] - 2s 200us/step - loss: 0.0836
Epoch 8/25
8000/8000 [==============================] - 2s 203us/step - loss: 0.0834
Epoch 9/25
8000/8000 [==============================] - 2s 202us/step - loss: 0.0832
Epoch 10/25
8000/8000 [==============================] - 2s 200us/step - loss: 0.0831
Epoch 11/25
8000/8000 [==============================] - 2s 203us/step - loss: 0.0830
Epoch 12/25
8000/8000 [==============================] - 2s 205us/step - loss: 0.0828
Epoch 13/25
8000/8000 [==============================] - 2s 202us/step - loss: 0.0828
Epoch 14/25
8000/8000 [==============================] - 2s 200us/step - loss: 0.0827
Epoch 15/25
8000/8000 [==============================] - 2s 200us/step - loss: 0.0826
Epoch 16/25
8000/8000 [==============================] - 2s 202us/step - loss: 0.0826
Epoch 17/25
8000/8000 [==============================] - 2s 199us/step - loss: 0.0825
Epoch 18/25
8000/8000 [==============================] - 2s 203us/step - loss: 0.0825
Epoch 19/25
8000/8000 [==============================] - 2s 199us/step - loss: 0.0825
Epoch 20/25
8000/8000 [==============================] - 2s 205us/step - loss: 0.0825
Epoch 21/25
8000/8000 [==============================] - 2s 201us/step - loss: 0.0824
Epoch 22/25
8000/8000 [==============================] - 2s 202us/step - loss: 0.0824
Epoch 23/25
8000/8000 [==============================] - 2s 203us/step - loss: 0.0824
Epoch 24/25
8000/8000 [==============================] - 2s 197us/step - loss: 0.0824
Epoch 25/25
8000/8000 [==============================] - 2s 197us/step - loss: 0.0824
mse ev: 0.055818; mse rr: 0.072826 (0.075059); mse simec (0hl): 0.071113 (0.074487); mse simec (2hl): 0.057224 (0.064911); mse simec (5hl): 0.056174 (0.063413)
4
Epoch 1/25
8000/8000 [==============================] - 1s 185us/step - loss: 0.1209
Epoch 2/25
8000/8000 [==============================] - 1s 140us/step - loss: 0.1054
Epoch 3/25
8000/8000 [==============================] - 1s 140us/step - loss: 0.1033
Epoch 4/25
8000/8000 [==============================] - 1s 140us/step - loss: 0.1021
Epoch 5/25
8000/8000 [==============================] - 1s 146us/step - loss: 0.1009
Epoch 6/25
8000/8000 [==============================] - 1s 142us/step - loss: 0.0996
Epoch 7/25
8000/8000 [==============================] - 1s 143us/step - loss: 0.0978
Epoch 8/25
8000/8000 [==============================] - 1s 142us/step - loss: 0.0953
Epoch 9/25
8000/8000 [==============================] - 1s 138us/step - loss: 0.0921
Epoch 10/25
8000/8000 [==============================] - 1s 140us/step - loss: 0.0885
Epoch 11/25
8000/8000 [==============================] - 1s 140us/step - loss: 0.0851
Epoch 12/25
8000/8000 [==============================] - 1s 140us/step - loss: 0.0827
Epoch 13/25
8000/8000 [==============================] - 1s 141us/step - loss: 0.0814
Epoch 14/25
8000/8000 [==============================] - 1s 137us/step - loss: 0.0809
Epoch 15/25
8000/8000 [==============================] - 1s 144us/step - loss: 0.0806
Epoch 16/25
8000/8000 [==============================] - 1s 142us/step - loss: 0.0805
Epoch 17/25
8000/8000 [==============================] - 1s 143us/step - loss: 0.0803
Epoch 18/25
8000/8000 [==============================] - 1s 144us/step - loss: 0.0802
Epoch 19/25
8000/8000 [==============================] - 1s 144us/step - loss: 0.0801
Epoch 20/25
8000/8000 [==============================] - 1s 142us/step - loss: 0.0801
Epoch 21/25
8000/8000 [==============================] - 1s 137us/step - loss: 0.0800
Epoch 22/25
8000/8000 [==============================] - 1s 143us/step - loss: 0.0800
Epoch 23/25
8000/8000 [==============================] - 1s 143us/step - loss: 0.0799
Epoch 24/25
8000/8000 [==============================] - 1s 139us/step - loss: 0.0799
Epoch 25/25
8000/8000 [==============================] - 1s 141us/step - loss: 0.0798
Epoch 1/25
8000/8000 [==============================] - 2s 214us/step - loss: 0.1048
Epoch 2/25
8000/8000 [==============================] - 1s 167us/step - loss: 0.0913
Epoch 3/25
8000/8000 [==============================] - 1s 167us/step - loss: 0.0791
Epoch 4/25
8000/8000 [==============================] - 1s 169us/step - loss: 0.0720
Epoch 5/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0708
Epoch 6/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0702
Epoch 7/25
8000/8000 [==============================] - 1s 168us/step - loss: 0.0698
Epoch 8/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0696
Epoch 9/25
8000/8000 [==============================] - 1s 167us/step - loss: 0.0693
Epoch 10/25
8000/8000 [==============================] - 1s 171us/step - loss: 0.0691
Epoch 11/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0689
Epoch 12/25
8000/8000 [==============================] - 1s 167us/step - loss: 0.0687
Epoch 13/25
8000/8000 [==============================] - 1s 168us/step - loss: 0.0685
Epoch 14/25
8000/8000 [==============================] - 1s 168us/step - loss: 0.0683
Epoch 15/25
8000/8000 [==============================] - 1s 172us/step - loss: 0.0682
Epoch 16/25
8000/8000 [==============================] - 1s 170us/step - loss: 0.0681
Epoch 17/25
8000/8000 [==============================] - 1s 167us/step - loss: 0.0680
Epoch 18/25
8000/8000 [==============================] - 1s 168us/step - loss: 0.0679
Epoch 19/25
8000/8000 [==============================] - 1s 170us/step - loss: 0.0679
Epoch 20/25
8000/8000 [==============================] - 1s 170us/step - loss: 0.0678
Epoch 21/25
8000/8000 [==============================] - 1s 169us/step - loss: 0.0677
Epoch 22/25
8000/8000 [==============================] - 1s 167us/step - loss: 0.0676
Epoch 23/25
8000/8000 [==============================] - 1s 171us/step - loss: 0.0676
Epoch 24/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0675
Epoch 25/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0674
Epoch 1/25
8000/8000 [==============================] - 2s 265us/step - loss: 0.1060
Epoch 2/25
8000/8000 [==============================] - 2s 202us/step - loss: 0.0903
Epoch 3/25
8000/8000 [==============================] - 2s 210us/step - loss: 0.0773
Epoch 4/25
8000/8000 [==============================] - 2s 204us/step - loss: 0.0708
Epoch 5/25
8000/8000 [==============================] - 2s 205us/step - loss: 0.0693
Epoch 6/25
8000/8000 [==============================] - 2s 203us/step - loss: 0.0686
Epoch 7/25
8000/8000 [==============================] - 2s 204us/step - loss: 0.0681
Epoch 8/25
8000/8000 [==============================] - 2s 203us/step - loss: 0.0677
Epoch 9/25
8000/8000 [==============================] - 2s 204us/step - loss: 0.0674
Epoch 10/25
8000/8000 [==============================] - 2s 207us/step - loss: 0.0672
Epoch 11/25
8000/8000 [==============================] - 2s 206us/step - loss: 0.0671
Epoch 12/25
8000/8000 [==============================] - 2s 206us/step - loss: 0.0669
Epoch 13/25
8000/8000 [==============================] - 2s 204us/step - loss: 0.0668
Epoch 14/25
8000/8000 [==============================] - 2s 206us/step - loss: 0.0667
Epoch 15/25
8000/8000 [==============================] - 2s 205us/step - loss: 0.0667
Epoch 16/25
8000/8000 [==============================] - 2s 206us/step - loss: 0.0666
Epoch 17/25
8000/8000 [==============================] - 2s 207us/step - loss: 0.0665
Epoch 18/25
8000/8000 [==============================] - 2s 204us/step - loss: 0.0665
Epoch 19/25
8000/8000 [==============================] - 2s 203us/step - loss: 0.0665
Epoch 20/25
8000/8000 [==============================] - 2s 207us/step - loss: 0.0664
Epoch 21/25
8000/8000 [==============================] - 2s 206us/step - loss: 0.0664
Epoch 22/25
8000/8000 [==============================] - 2s 206us/step - loss: 0.0663
Epoch 23/25
8000/8000 [==============================] - 2s 205us/step - loss: 0.0663
Epoch 24/25
8000/8000 [==============================] - 2s 203us/step - loss: 0.0663
Epoch 25/25
8000/8000 [==============================] - 2s 200us/step - loss: 0.0662
mse ev: 0.045566; mse rr: 0.068304 (0.071994); mse simec (0hl): 0.067147 (0.071882); mse simec (2hl): 0.047863 (0.058644); mse simec (5hl): 0.046097 (0.056709)
5
Epoch 1/25
8000/8000 [==============================] - 2s 196us/step - loss: 0.1195
Epoch 2/25
8000/8000 [==============================] - 1s 143us/step - loss: 0.1005
Epoch 3/25
8000/8000 [==============================] - 1s 142us/step - loss: 0.0975
Epoch 4/25
8000/8000 [==============================] - 1s 138us/step - loss: 0.0960
Epoch 5/25
8000/8000 [==============================] - 1s 137us/step - loss: 0.0946
Epoch 6/25
8000/8000 [==============================] - 1s 136us/step - loss: 0.0931
Epoch 7/25
8000/8000 [==============================] - 1s 139us/step - loss: 0.0911
Epoch 8/25
8000/8000 [==============================] - 1s 140us/step - loss: 0.0885
Epoch 9/25
8000/8000 [==============================] - 1s 141us/step - loss: 0.0850
Epoch 10/25
8000/8000 [==============================] - 1s 141us/step - loss: 0.0809
Epoch 11/25
8000/8000 [==============================] - 1s 139us/step - loss: 0.0768
Epoch 12/25
8000/8000 [==============================] - 1s 144us/step - loss: 0.0734
Epoch 13/25
8000/8000 [==============================] - 1s 141us/step - loss: 0.0712
Epoch 14/25
8000/8000 [==============================] - 1s 142us/step - loss: 0.0702
Epoch 15/25
8000/8000 [==============================] - 1s 143us/step - loss: 0.0697
Epoch 16/25
8000/8000 [==============================] - 1s 145us/step - loss: 0.0695
Epoch 17/25
8000/8000 [==============================] - 1s 144us/step - loss: 0.0693
Epoch 18/25
8000/8000 [==============================] - 1s 142us/step - loss: 0.0692
Epoch 19/25
8000/8000 [==============================] - 1s 140us/step - loss: 0.0691
Epoch 20/25
8000/8000 [==============================] - 1s 143us/step - loss: 0.0689
Epoch 21/25
8000/8000 [==============================] - 1s 137us/step - loss: 0.0689
Epoch 22/25
8000/8000 [==============================] - 1s 141us/step - loss: 0.0688
Epoch 23/25
8000/8000 [==============================] - 1s 140us/step - loss: 0.0688
Epoch 24/25
8000/8000 [==============================] - 1s 144us/step - loss: 0.0687
Epoch 25/25
8000/8000 [==============================] - 1s 138us/step - loss: 0.0687
Epoch 1/25
8000/8000 [==============================] - 2s 222us/step - loss: 0.0999
Epoch 2/25
8000/8000 [==============================] - 1s 168us/step - loss: 0.0817
Epoch 3/25
8000/8000 [==============================] - 1s 171us/step - loss: 0.0682
Epoch 4/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0596
Epoch 5/25
8000/8000 [==============================] - 1s 170us/step - loss: 0.0574
Epoch 6/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0565
Epoch 7/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0560
Epoch 8/25
8000/8000 [==============================] - 1s 169us/step - loss: 0.0556
Epoch 9/25
8000/8000 [==============================] - 1s 169us/step - loss: 0.0552
Epoch 10/25
8000/8000 [==============================] - 1s 171us/step - loss: 0.0549
Epoch 11/25
8000/8000 [==============================] - 1s 167us/step - loss: 0.0546
Epoch 12/25
8000/8000 [==============================] - 1s 167us/step - loss: 0.0544
Epoch 13/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0541
Epoch 14/25
8000/8000 [==============================] - 1s 168us/step - loss: 0.0540
Epoch 15/25
8000/8000 [==============================] - 1s 168us/step - loss: 0.0538
Epoch 16/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0537
Epoch 17/25
8000/8000 [==============================] - 1s 175us/step - loss: 0.0535
Epoch 18/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0534
Epoch 19/25
8000/8000 [==============================] - 1s 168us/step - loss: 0.0533
Epoch 20/25
8000/8000 [==============================] - 1s 167us/step - loss: 0.0531
Epoch 21/25
8000/8000 [==============================] - 1s 167us/step - loss: 0.0530
Epoch 22/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0529
Epoch 23/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0528
Epoch 24/25
8000/8000 [==============================] - 1s 167us/step - loss: 0.0528
Epoch 25/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0527
Epoch 1/25
8000/8000 [==============================] - 2s 267us/step - loss: 0.0970
Epoch 2/25
8000/8000 [==============================] - 2s 202us/step - loss: 0.0771
Epoch 3/25
8000/8000 [==============================] - 2s 204us/step - loss: 0.0614
Epoch 4/25
8000/8000 [==============================] - 2s 202us/step - loss: 0.0559
Epoch 5/25
8000/8000 [==============================] - 2s 204us/step - loss: 0.0548
Epoch 6/25
8000/8000 [==============================] - 2s 206us/step - loss: 0.0542
Epoch 7/25
8000/8000 [==============================] - 2s 200us/step - loss: 0.0537
Epoch 8/25
8000/8000 [==============================] - 2s 202us/step - loss: 0.0533
Epoch 9/25
8000/8000 [==============================] - 2s 201us/step - loss: 0.0531
Epoch 10/25
8000/8000 [==============================] - 2s 205us/step - loss: 0.0528
Epoch 11/25
8000/8000 [==============================] - 2s 203us/step - loss: 0.0526
Epoch 12/25
8000/8000 [==============================] - 2s 200us/step - loss: 0.0524
Epoch 13/25
8000/8000 [==============================] - 2s 200us/step - loss: 0.0523
Epoch 14/25
8000/8000 [==============================] - 2s 202us/step - loss: 0.0522
Epoch 15/25
8000/8000 [==============================] - 2s 203us/step - loss: 0.0521
Epoch 16/25
8000/8000 [==============================] - 2s 205us/step - loss: 0.0520
Epoch 17/25
8000/8000 [==============================] - 2s 201us/step - loss: 0.0519
Epoch 18/25
8000/8000 [==============================] - 2s 204us/step - loss: 0.0518
Epoch 19/25
8000/8000 [==============================] - 2s 207us/step - loss: 0.0517
Epoch 20/25
8000/8000 [==============================] - 2s 201us/step - loss: 0.0517
Epoch 21/25
8000/8000 [==============================] - 2s 203us/step - loss: 0.0517
Epoch 22/25
8000/8000 [==============================] - 2s 198us/step - loss: 0.0516
Epoch 23/25
8000/8000 [==============================] - 2s 202us/step - loss: 0.0516
Epoch 24/25
8000/8000 [==============================] - 2s 200us/step - loss: 0.0515
Epoch 25/25
8000/8000 [==============================] - 2s 206us/step - loss: 0.0515
mse ev: 0.035617; mse rr: 0.064545 (0.069153); mse simec (0hl): 0.063182 (0.068509); mse simec (2hl): 0.038890 (0.051735); mse simec (5hl): 0.036935 (0.051086)
6
Epoch 1/25
8000/8000 [==============================] - 2s 203us/step - loss: 0.1185
Epoch 2/25
8000/8000 [==============================] - 1s 144us/step - loss: 0.0970
Epoch 3/25
8000/8000 [==============================] - 1s 141us/step - loss: 0.0926
Epoch 4/25
8000/8000 [==============================] - 1s 147us/step - loss: 0.0906
Epoch 5/25
8000/8000 [==============================] - 1s 141us/step - loss: 0.0887
Epoch 6/25
8000/8000 [==============================] - 1s 136us/step - loss: 0.0864
Epoch 7/25
8000/8000 [==============================] - 1s 145us/step - loss: 0.0833
Epoch 8/25
8000/8000 [==============================] - 1s 144us/step - loss: 0.0795
Epoch 9/25
8000/8000 [==============================] - 1s 142us/step - loss: 0.0753
Epoch 10/25
8000/8000 [==============================] - 1s 141us/step - loss: 0.0711
Epoch 11/25
8000/8000 [==============================] - 1s 142us/step - loss: 0.0675
Epoch 12/25
8000/8000 [==============================] - 1s 142us/step - loss: 0.0647
Epoch 13/25
8000/8000 [==============================] - 1s 146us/step - loss: 0.0628
Epoch 14/25
8000/8000 [==============================] - 1s 144us/step - loss: 0.0617
Epoch 15/25
8000/8000 [==============================] - 1s 141us/step - loss: 0.0611
Epoch 16/25
8000/8000 [==============================] - 1s 141us/step - loss: 0.0608
Epoch 17/25
8000/8000 [==============================] - 1s 140us/step - loss: 0.0605
Epoch 18/25
8000/8000 [==============================] - 1s 147us/step - loss: 0.0603
Epoch 19/25
8000/8000 [==============================] - 1s 143us/step - loss: 0.0602
Epoch 20/25
8000/8000 [==============================] - 1s 145us/step - loss: 0.0600
Epoch 21/25
8000/8000 [==============================] - 1s 142us/step - loss: 0.0599
Epoch 22/25
8000/8000 [==============================] - 1s 146us/step - loss: 0.0598
Epoch 23/25
8000/8000 [==============================] - 1s 141us/step - loss: 0.0597
Epoch 24/25
8000/8000 [==============================] - 1s 146us/step - loss: 0.0596
Epoch 25/25
8000/8000 [==============================] - 1s 149us/step - loss: 0.0595
Epoch 1/25
8000/8000 [==============================] - 2s 238us/step - loss: 0.0956
Epoch 2/25
8000/8000 [==============================] - 1s 172us/step - loss: 0.0723
Epoch 3/25
8000/8000 [==============================] - 1s 170us/step - loss: 0.0573
Epoch 4/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0497
Epoch 5/25
8000/8000 [==============================] - 1s 167us/step - loss: 0.0473
Epoch 6/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0461
Epoch 7/25
8000/8000 [==============================] - 1s 170us/step - loss: 0.0454
Epoch 8/25
8000/8000 [==============================] - 1s 168us/step - loss: 0.0448
Epoch 9/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0443
Epoch 10/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0439
Epoch 11/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0435
Epoch 12/25
8000/8000 [==============================] - 1s 168us/step - loss: 0.0433
Epoch 13/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0430
Epoch 14/25
8000/8000 [==============================] - 1s 167us/step - loss: 0.0427
Epoch 15/25
8000/8000 [==============================] - 1s 171us/step - loss: 0.0425
Epoch 16/25
8000/8000 [==============================] - 1s 169us/step - loss: 0.0422
Epoch 17/25
8000/8000 [==============================] - 1s 168us/step - loss: 0.0420
Epoch 18/25
8000/8000 [==============================] - 1s 169us/step - loss: 0.0418
Epoch 19/25
8000/8000 [==============================] - 1s 169us/step - loss: 0.0416
Epoch 20/25
8000/8000 [==============================] - 1s 168us/step - loss: 0.0415
Epoch 21/25
8000/8000 [==============================] - 1s 167us/step - loss: 0.0412
Epoch 22/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0411
Epoch 23/25
8000/8000 [==============================] - 1s 172us/step - loss: 0.0409
Epoch 24/25
8000/8000 [==============================] - 1s 167us/step - loss: 0.0408
Epoch 25/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0406
Epoch 1/25
8000/8000 [==============================] - 2s 294us/step - loss: 0.0991
Epoch 2/25
8000/8000 [==============================] - 2s 198us/step - loss: 0.0673
Epoch 3/25
8000/8000 [==============================] - 2s 202us/step - loss: 0.0509
Epoch 4/25
8000/8000 [==============================] - 2s 200us/step - loss: 0.0441
Epoch 5/25
8000/8000 [==============================] - 2s 201us/step - loss: 0.0427
Epoch 6/25
8000/8000 [==============================] - 2s 200us/step - loss: 0.0420
Epoch 7/25
8000/8000 [==============================] - 2s 200us/step - loss: 0.0414
Epoch 8/25
8000/8000 [==============================] - 2s 197us/step - loss: 0.0409
Epoch 9/25
8000/8000 [==============================] - 2s 204us/step - loss: 0.0405
Epoch 10/25
8000/8000 [==============================] - 2s 200us/step - loss: 0.0401
Epoch 11/25
8000/8000 [==============================] - 2s 201us/step - loss: 0.0397
Epoch 12/25
8000/8000 [==============================] - 2s 198us/step - loss: 0.0394
Epoch 13/25
8000/8000 [==============================] - 2s 200us/step - loss: 0.0391
Epoch 14/25
8000/8000 [==============================] - 2s 198us/step - loss: 0.0389
Epoch 15/25
8000/8000 [==============================] - 2s 200us/step - loss: 0.0386
Epoch 16/25
8000/8000 [==============================] - 2s 204us/step - loss: 0.0384
Epoch 17/25
8000/8000 [==============================] - 2s 202us/step - loss: 0.0382
Epoch 18/25
8000/8000 [==============================] - 2s 198us/step - loss: 0.0381
Epoch 19/25
8000/8000 [==============================] - 2s 196us/step - loss: 0.0380
Epoch 20/25
8000/8000 [==============================] - 2s 201us/step - loss: 0.0379
Epoch 21/25
8000/8000 [==============================] - 2s 201us/step - loss: 0.0378
Epoch 22/25
8000/8000 [==============================] - 2s 199us/step - loss: 0.0378
Epoch 23/25
8000/8000 [==============================] - 2s 194us/step - loss: 0.0377
Epoch 24/25
8000/8000 [==============================] - 2s 199us/step - loss: 0.0376
Epoch 25/25
8000/8000 [==============================] - 2s 207us/step - loss: 0.0375
mse ev: 0.025953; mse rr: 0.062133 (0.066742); mse simec (0hl): 0.060091 (0.064931); mse simec (2hl): 0.032779 (0.047312); mse simec (5hl): 0.027733 (0.044223)
7
Epoch 1/25
8000/8000 [==============================] - 2s 214us/step - loss: 0.1154
Epoch 2/25
8000/8000 [==============================] - 1s 149us/step - loss: 0.0950
Epoch 3/25
8000/8000 [==============================] - 1s 145us/step - loss: 0.0895
Epoch 4/25
8000/8000 [==============================] - 1s 143us/step - loss: 0.0861
Epoch 5/25
8000/8000 [==============================] - 1s 144us/step - loss: 0.0833
Epoch 6/25
8000/8000 [==============================] - 1s 144us/step - loss: 0.0802
Epoch 7/25
8000/8000 [==============================] - 1s 146us/step - loss: 0.0762
Epoch 8/25
8000/8000 [==============================] - 1s 143us/step - loss: 0.0713
Epoch 9/25
8000/8000 [==============================] - 1s 141us/step - loss: 0.0664
Epoch 10/25
8000/8000 [==============================] - 1s 146us/step - loss: 0.0620
Epoch 11/25
8000/8000 [==============================] - 1s 144us/step - loss: 0.0585
Epoch 12/25
8000/8000 [==============================] - 1s 145us/step - loss: 0.0560
Epoch 13/25
8000/8000 [==============================] - 1s 145us/step - loss: 0.0545
Epoch 14/25
8000/8000 [==============================] - 1s 144us/step - loss: 0.0535
Epoch 15/25
8000/8000 [==============================] - 1s 148us/step - loss: 0.0530
Epoch 16/25
8000/8000 [==============================] - 1s 143us/step - loss: 0.0527
Epoch 17/25
8000/8000 [==============================] - 1s 142us/step - loss: 0.0524
Epoch 18/25
8000/8000 [==============================] - 1s 140us/step - loss: 0.0523
Epoch 19/25
8000/8000 [==============================] - 1s 146us/step - loss: 0.0521
Epoch 20/25
8000/8000 [==============================] - 1s 143us/step - loss: 0.0520
Epoch 21/25
8000/8000 [==============================] - 1s 144us/step - loss: 0.0519
Epoch 22/25
8000/8000 [==============================] - 1s 144us/step - loss: 0.0517
Epoch 23/25
8000/8000 [==============================] - 1s 145us/step - loss: 0.0516
Epoch 24/25
8000/8000 [==============================] - 1s 144us/step - loss: 0.0516
Epoch 25/25
8000/8000 [==============================] - 1s 147us/step - loss: 0.0515
Epoch 1/25
8000/8000 [==============================] - 2s 245us/step - loss: 0.0918
Epoch 2/25
8000/8000 [==============================] - 1s 170us/step - loss: 0.0633
Epoch 3/25
8000/8000 [==============================] - 1s 167us/step - loss: 0.0460
Epoch 4/25
8000/8000 [==============================] - 1s 168us/step - loss: 0.0381
Epoch 5/25
8000/8000 [==============================] - 1s 170us/step - loss: 0.0361
Epoch 6/25
8000/8000 [==============================] - 1s 171us/step - loss: 0.0351
Epoch 7/25
8000/8000 [==============================] - 1s 172us/step - loss: 0.0345
Epoch 8/25
8000/8000 [==============================] - 1s 171us/step - loss: 0.0338
Epoch 9/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0333
Epoch 10/25
8000/8000 [==============================] - 1s 169us/step - loss: 0.0329
Epoch 11/25
8000/8000 [==============================] - 1s 172us/step - loss: 0.0325
Epoch 12/25
8000/8000 [==============================] - 1s 171us/step - loss: 0.0321
Epoch 13/25
8000/8000 [==============================] - 1s 169us/step - loss: 0.0318
Epoch 14/25
8000/8000 [==============================] - 1s 167us/step - loss: 0.0314
Epoch 15/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0311
Epoch 16/25
8000/8000 [==============================] - 1s 170us/step - loss: 0.0308
Epoch 17/25
8000/8000 [==============================] - 1s 173us/step - loss: 0.0305
Epoch 18/25
8000/8000 [==============================] - 1s 172us/step - loss: 0.0303
Epoch 19/25
8000/8000 [==============================] - 1s 169us/step - loss: 0.0300
Epoch 20/25
8000/8000 [==============================] - 1s 170us/step - loss: 0.0298
Epoch 21/25
8000/8000 [==============================] - 1s 169us/step - loss: 0.0297
Epoch 22/25
8000/8000 [==============================] - 1s 168us/step - loss: 0.0294
Epoch 23/25
8000/8000 [==============================] - 1s 168us/step - loss: 0.0293
Epoch 24/25
8000/8000 [==============================] - 1s 169us/step - loss: 0.0292
Epoch 25/25
8000/8000 [==============================] - 1s 170us/step - loss: 0.0290
Epoch 1/25
8000/8000 [==============================] - 2s 287us/step - loss: 0.0962
Epoch 2/25
8000/8000 [==============================] - 2s 202us/step - loss: 0.0596
Epoch 3/25
8000/8000 [==============================] - 2s 206us/step - loss: 0.0403
Epoch 4/25
8000/8000 [==============================] - 2s 209us/step - loss: 0.0338
Epoch 5/25
8000/8000 [==============================] - 2s 206us/step - loss: 0.0324
Epoch 6/25
8000/8000 [==============================] - 2s 203us/step - loss: 0.0315
Epoch 7/25
8000/8000 [==============================] - 2s 207us/step - loss: 0.0307
Epoch 8/25
8000/8000 [==============================] - 2s 202us/step - loss: 0.0301
Epoch 9/25
8000/8000 [==============================] - 2s 201us/step - loss: 0.0296
Epoch 10/25
8000/8000 [==============================] - 2s 203us/step - loss: 0.0290
Epoch 11/25
8000/8000 [==============================] - 2s 203us/step - loss: 0.0286
Epoch 12/25
8000/8000 [==============================] - 2s 205us/step - loss: 0.0282
Epoch 13/25
8000/8000 [==============================] - 2s 206us/step - loss: 0.0279
Epoch 14/25
8000/8000 [==============================] - 2s 207us/step - loss: 0.0276
Epoch 15/25
8000/8000 [==============================] - 2s 210us/step - loss: 0.0273
Epoch 16/25
8000/8000 [==============================] - 2s 209us/step - loss: 0.0270
Epoch 17/25
8000/8000 [==============================] - 2s 200us/step - loss: 0.0269
Epoch 18/25
8000/8000 [==============================] - 2s 203us/step - loss: 0.0267
Epoch 19/25
8000/8000 [==============================] - 2s 201us/step - loss: 0.0265
Epoch 20/25
8000/8000 [==============================] - 2s 202us/step - loss: 0.0263
Epoch 21/25
8000/8000 [==============================] - 2s 205us/step - loss: 0.0261
Epoch 22/25
8000/8000 [==============================] - 2s 200us/step - loss: 0.0260
Epoch 23/25
8000/8000 [==============================] - 2s 203us/step - loss: 0.0260
Epoch 24/25
8000/8000 [==============================] - 2s 205us/step - loss: 0.0259
Epoch 25/25
8000/8000 [==============================] - 2s 206us/step - loss: 0.0258
mse ev: 0.016659; mse rr: 0.059272 (0.064194); mse simec (0hl): 0.058145 (0.063481); mse simec (2hl): 0.025567 (0.041186); mse simec (5hl): 0.019750 (0.038146)
8
Epoch 1/25
8000/8000 [==============================] - 2s 220us/step - loss: 0.1148
Epoch 2/25
8000/8000 [==============================] - 1s 145us/step - loss: 0.0944
Epoch 3/25
8000/8000 [==============================] - 1s 144us/step - loss: 0.0891
Epoch 4/25
8000/8000 [==============================] - 1s 140us/step - loss: 0.0854
Epoch 5/25
8000/8000 [==============================] - 1s 145us/step - loss: 0.0820
Epoch 6/25
8000/8000 [==============================] - 1s 143us/step - loss: 0.0790
Epoch 7/25
8000/8000 [==============================] - 1s 141us/step - loss: 0.0757
Epoch 8/25
8000/8000 [==============================] - 1s 144us/step - loss: 0.0712
Epoch 9/25
8000/8000 [==============================] - 1s 143us/step - loss: 0.0652
Epoch 10/25
8000/8000 [==============================] - 1s 141us/step - loss: 0.0587
Epoch 11/25
8000/8000 [==============================] - 1s 143us/step - loss: 0.0530
Epoch 12/25
8000/8000 [==============================] - 1s 139us/step - loss: 0.0494
Epoch 13/25
8000/8000 [==============================] - 1s 141us/step - loss: 0.0475
Epoch 14/25
8000/8000 [==============================] - 1s 143us/step - loss: 0.0465
Epoch 15/25
8000/8000 [==============================] - 1s 140us/step - loss: 0.0461
Epoch 16/25
8000/8000 [==============================] - 1s 148us/step - loss: 0.0457
Epoch 17/25
8000/8000 [==============================] - 1s 146us/step - loss: 0.0455
Epoch 18/25
8000/8000 [==============================] - 1s 146us/step - loss: 0.0453
Epoch 19/25
8000/8000 [==============================] - 1s 145us/step - loss: 0.0451
Epoch 20/25
8000/8000 [==============================] - 1s 141us/step - loss: 0.0449
Epoch 21/25
8000/8000 [==============================] - 1s 144us/step - loss: 0.0448
Epoch 22/25
8000/8000 [==============================] - 1s 141us/step - loss: 0.0447
Epoch 23/25
8000/8000 [==============================] - 1s 145us/step - loss: 0.0446
Epoch 24/25
8000/8000 [==============================] - 1s 143us/step - loss: 0.0445
Epoch 25/25
8000/8000 [==============================] - 1s 144us/step - loss: 0.0444
Epoch 1/25
8000/8000 [==============================] - 2s 258us/step - loss: 0.0899
Epoch 2/25
8000/8000 [==============================] - 1s 169us/step - loss: 0.0613
Epoch 3/25
8000/8000 [==============================] - 1s 167us/step - loss: 0.0388
Epoch 4/25
8000/8000 [==============================] - 1s 171us/step - loss: 0.0294
Epoch 5/25
8000/8000 [==============================] - 1s 169us/step - loss: 0.0274
Epoch 6/25
8000/8000 [==============================] - 1s 171us/step - loss: 0.0262
Epoch 7/25
8000/8000 [==============================] - 1s 171us/step - loss: 0.0253
Epoch 8/25
8000/8000 [==============================] - 1s 169us/step - loss: 0.0246
Epoch 9/25
8000/8000 [==============================] - 1s 171us/step - loss: 0.0238
Epoch 10/25
8000/8000 [==============================] - 1s 171us/step - loss: 0.0233
Epoch 11/25
8000/8000 [==============================] - 1s 171us/step - loss: 0.0227
Epoch 12/25
8000/8000 [==============================] - 1s 174us/step - loss: 0.0223
Epoch 13/25
8000/8000 [==============================] - 1s 169us/step - loss: 0.0218
Epoch 14/25
8000/8000 [==============================] - 1s 170us/step - loss: 0.0215
Epoch 15/25
8000/8000 [==============================] - 1s 173us/step - loss: 0.0211
Epoch 16/25
8000/8000 [==============================] - 1s 169us/step - loss: 0.0207
Epoch 17/25
8000/8000 [==============================] - 1s 172us/step - loss: 0.0205
Epoch 18/25
8000/8000 [==============================] - 1s 174us/step - loss: 0.0201
Epoch 19/25
8000/8000 [==============================] - 1s 171us/step - loss: 0.0198
Epoch 20/25
8000/8000 [==============================] - 1s 171us/step - loss: 0.0196
Epoch 21/25
8000/8000 [==============================] - 1s 172us/step - loss: 0.0193
Epoch 22/25
8000/8000 [==============================] - 1s 169us/step - loss: 0.0191
Epoch 23/25
8000/8000 [==============================] - 1s 167us/step - loss: 0.0188
Epoch 24/25
8000/8000 [==============================] - 1s 170us/step - loss: 0.0186
Epoch 25/25
8000/8000 [==============================] - 1s 169us/step - loss: 0.0184
Epoch 1/25
8000/8000 [==============================] - 2s 296us/step - loss: 0.0931
Epoch 2/25
8000/8000 [==============================] - 2s 208us/step - loss: 0.0542
Epoch 3/25
8000/8000 [==============================] - 2s 208us/step - loss: 0.0301
Epoch 4/25
8000/8000 [==============================] - 2s 207us/step - loss: 0.0234
Epoch 5/25
8000/8000 [==============================] - 2s 209us/step - loss: 0.0216
Epoch 6/25
8000/8000 [==============================] - 2s 208us/step - loss: 0.0204
Epoch 7/25
8000/8000 [==============================] - 2s 207us/step - loss: 0.0193
Epoch 8/25
8000/8000 [==============================] - 2s 208us/step - loss: 0.0185
Epoch 9/25
8000/8000 [==============================] - 2s 208us/step - loss: 0.0178
Epoch 10/25
8000/8000 [==============================] - 2s 205us/step - loss: 0.0172
Epoch 11/25
8000/8000 [==============================] - 2s 207us/step - loss: 0.0166
Epoch 12/25
8000/8000 [==============================] - 2s 207us/step - loss: 0.0161
Epoch 13/25
8000/8000 [==============================] - 2s 203us/step - loss: 0.0157
Epoch 14/25
8000/8000 [==============================] - 2s 207us/step - loss: 0.0153
Epoch 15/25
8000/8000 [==============================] - 2s 209us/step - loss: 0.0150
Epoch 16/25
8000/8000 [==============================] - 2s 210us/step - loss: 0.0147
Epoch 17/25
8000/8000 [==============================] - 2s 209us/step - loss: 0.0144
Epoch 18/25
8000/8000 [==============================] - 2s 208us/step - loss: 0.0142
Epoch 19/25
8000/8000 [==============================] - 2s 209us/step - loss: 0.0140
Epoch 20/25
8000/8000 [==============================] - 2s 207us/step - loss: 0.0138
Epoch 21/25
8000/8000 [==============================] - 2s 212us/step - loss: 0.0137
Epoch 22/25
8000/8000 [==============================] - 2s 207us/step - loss: 0.0135
Epoch 23/25
8000/8000 [==============================] - 2s 204us/step - loss: 0.0134
Epoch 24/25
8000/8000 [==============================] - 2s 209us/step - loss: 0.0133
Epoch 25/25
8000/8000 [==============================] - 2s 207us/step - loss: 0.0131
mse ev: 0.008156; mse rr: 0.057297 (0.062027); mse simec (0hl): 0.056857 (0.062827); mse simec (2hl): 0.018794 (0.036313); mse simec (5hl): 0.011405 (0.033130)
9
Epoch 1/25
8000/8000 [==============================] - 2s 231us/step - loss: 0.1157
Epoch 2/25
8000/8000 [==============================] - 1s 142us/step - loss: 0.0922
Epoch 3/25
8000/8000 [==============================] - 1s 148us/step - loss: 0.0863
Epoch 4/25
8000/8000 [==============================] - 1s 144us/step - loss: 0.0831
Epoch 5/25
8000/8000 [==============================] - 1s 146us/step - loss: 0.0801
Epoch 6/25
8000/8000 [==============================] - 1s 144us/step - loss: 0.0762
Epoch 7/25
8000/8000 [==============================] - 1s 144us/step - loss: 0.0717
Epoch 8/25
8000/8000 [==============================] - 1s 144us/step - loss: 0.0660
Epoch 9/25
8000/8000 [==============================] - 1s 147us/step - loss: 0.0589
Epoch 10/25
8000/8000 [==============================] - 1s 151us/step - loss: 0.0516
Epoch 11/25
8000/8000 [==============================] - 1s 151us/step - loss: 0.0460
Epoch 12/25
8000/8000 [==============================] - 1s 146us/step - loss: 0.0427
Epoch 13/25
8000/8000 [==============================] - 1s 144us/step - loss: 0.0407
Epoch 14/25
8000/8000 [==============================] - 1s 146us/step - loss: 0.0396
Epoch 15/25
8000/8000 [==============================] - 1s 142us/step - loss: 0.0390
Epoch 16/25
8000/8000 [==============================] - 1s 150us/step - loss: 0.0386
Epoch 17/25
8000/8000 [==============================] - 1s 150us/step - loss: 0.0383
Epoch 18/25
8000/8000 [==============================] - 1s 149us/step - loss: 0.0381
Epoch 19/25
8000/8000 [==============================] - 1s 146us/step - loss: 0.0379
Epoch 20/25
8000/8000 [==============================] - 1s 148us/step - loss: 0.0378
Epoch 21/25
8000/8000 [==============================] - 1s 149us/step - loss: 0.0376
Epoch 22/25
8000/8000 [==============================] - 1s 146us/step - loss: 0.0375
Epoch 23/25
8000/8000 [==============================] - 1s 148us/step - loss: 0.0374
Epoch 24/25
8000/8000 [==============================] - 1s 142us/step - loss: 0.0374
Epoch 25/25
8000/8000 [==============================] - 1s 152us/step - loss: 0.0372
Epoch 1/25
8000/8000 [==============================] - 2s 264us/step - loss: 0.0897
Epoch 2/25
8000/8000 [==============================] - 1s 175us/step - loss: 0.0566
Epoch 3/25
8000/8000 [==============================] - 1s 169us/step - loss: 0.0308
Epoch 4/25
8000/8000 [==============================] - 1s 168us/step - loss: 0.0201
Epoch 5/25
8000/8000 [==============================] - 1s 168us/step - loss: 0.0180
Epoch 6/25
8000/8000 [==============================] - 1s 171us/step - loss: 0.0167
Epoch 7/25
8000/8000 [==============================] - 1s 171us/step - loss: 0.0157
Epoch 8/25
8000/8000 [==============================] - 1s 173us/step - loss: 0.0149
Epoch 9/25
8000/8000 [==============================] - 1s 172us/step - loss: 0.0141
Epoch 10/25
8000/8000 [==============================] - 1s 172us/step - loss: 0.0136
Epoch 11/25
8000/8000 [==============================] - 1s 175us/step - loss: 0.0129
Epoch 12/25
8000/8000 [==============================] - 1s 175us/step - loss: 0.0125
Epoch 13/25
8000/8000 [==============================] - 1s 170us/step - loss: 0.0119
Epoch 14/25
8000/8000 [==============================] - 1s 179us/step - loss: 0.0116
Epoch 15/25
8000/8000 [==============================] - 1s 177us/step - loss: 0.0111
Epoch 16/25
8000/8000 [==============================] - 1s 175us/step - loss: 0.0108
Epoch 17/25
8000/8000 [==============================] - 1s 172us/step - loss: 0.0105
Epoch 18/25
8000/8000 [==============================] - 1s 175us/step - loss: 0.0101
Epoch 19/25
8000/8000 [==============================] - 1s 173us/step - loss: 0.0099
Epoch 20/25
8000/8000 [==============================] - 1s 173us/step - loss: 0.0096
Epoch 21/25
8000/8000 [==============================] - 1s 176us/step - loss: 0.0094
Epoch 22/25
8000/8000 [==============================] - 1s 171us/step - loss: 0.0091
Epoch 23/25
8000/8000 [==============================] - 1s 171us/step - loss: 0.0089
Epoch 24/25
8000/8000 [==============================] - 1s 176us/step - loss: 0.0086
Epoch 25/25
8000/8000 [==============================] - 1s 174us/step - loss: 0.0084
Epoch 1/25
8000/8000 [==============================] - 3s 314us/step - loss: 0.0938
Epoch 2/25
8000/8000 [==============================] - 2s 211us/step - loss: 0.0575
Epoch 3/25
8000/8000 [==============================] - 2s 210us/step - loss: 0.0281
Epoch 4/25
8000/8000 [==============================] - 2s 211us/step - loss: 0.0139
Epoch 5/25
8000/8000 [==============================] - 2s 214us/step - loss: 0.0109
Epoch 6/25
8000/8000 [==============================] - 2s 207us/step - loss: 0.0095
Epoch 7/25
8000/8000 [==============================] - 2s 209us/step - loss: 0.0082
Epoch 8/25
8000/8000 [==============================] - 2s 209us/step - loss: 0.0071
Epoch 9/25
8000/8000 [==============================] - 2s 214us/step - loss: 0.0062
Epoch 10/25
8000/8000 [==============================] - 2s 211us/step - loss: 0.0056
Epoch 11/25
8000/8000 [==============================] - 2s 213us/step - loss: 0.0049
Epoch 12/25
8000/8000 [==============================] - 2s 212us/step - loss: 0.0043
Epoch 13/25
8000/8000 [==============================] - 2s 213us/step - loss: 0.0039
Epoch 14/25
8000/8000 [==============================] - 2s 210us/step - loss: 0.0034
Epoch 15/25
8000/8000 [==============================] - 2s 210us/step - loss: 0.0032
Epoch 16/25
8000/8000 [==============================] - 2s 205us/step - loss: 0.0028
Epoch 17/25
8000/8000 [==============================] - 2s 209us/step - loss: 0.0026
Epoch 18/25
8000/8000 [==============================] - 2s 210us/step - loss: 0.0025
Epoch 19/25
8000/8000 [==============================] - 2s 207us/step - loss: 0.0022
Epoch 20/25
8000/8000 [==============================] - 2s 204us/step - loss: 0.0020
Epoch 21/25
8000/8000 [==============================] - 2s 215us/step - loss: 0.0020
Epoch 22/25
8000/8000 [==============================] - 2s 208us/step - loss: 0.0019
Epoch 23/25
8000/8000 [==============================] - 2s 213us/step - loss: 0.0017
Epoch 24/25
8000/8000 [==============================] - 2s 212us/step - loss: 0.0017
Epoch 25/25
8000/8000 [==============================] - 2s 208us/step - loss: 0.0016
mse ev: 0.000000; mse rr: 0.055404 (0.060183); mse simec (0hl): 0.055934 (0.062521); mse simec (2hl): 0.014201 (0.031433); mse simec (5hl): 0.002482 (0.025714)
10
Epoch 1/25
8000/8000 [==============================] - 2s 251us/step - loss: 0.1137
Epoch 2/25
8000/8000 [==============================] - 1s 148us/step - loss: 0.0913
Epoch 3/25
8000/8000 [==============================] - 1s 147us/step - loss: 0.0860
Epoch 4/25
8000/8000 [==============================] - 1s 148us/step - loss: 0.0826
Epoch 5/25
8000/8000 [==============================] - 1s 148us/step - loss: 0.0794
Epoch 6/25
8000/8000 [==============================] - 1s 142us/step - loss: 0.0763
Epoch 7/25
8000/8000 [==============================] - 1s 143us/step - loss: 0.0728
Epoch 8/25
8000/8000 [==============================] - 1s 146us/step - loss: 0.0682
Epoch 9/25
8000/8000 [==============================] - 1s 150us/step - loss: 0.0623
Epoch 10/25
8000/8000 [==============================] - 1s 149us/step - loss: 0.0559
Epoch 11/25
8000/8000 [==============================] - ETA: 0s - loss: 0.050 - 1s 150us/step - loss: 0.0502
Epoch 12/25
8000/8000 [==============================] - 1s 149us/step - loss: 0.0464
Epoch 13/25
8000/8000 [==============================] - 1s 147us/step - loss: 0.0441
Epoch 14/25
8000/8000 [==============================] - 1s 147us/step - loss: 0.0427
Epoch 15/25
8000/8000 [==============================] - 1s 147us/step - loss: 0.0418
Epoch 16/25
8000/8000 [==============================] - 1s 148us/step - loss: 0.0412
Epoch 17/25
8000/8000 [==============================] - 1s 147us/step - loss: 0.0406
Epoch 18/25
8000/8000 [==============================] - 1s 149us/step - loss: 0.0400
Epoch 19/25
8000/8000 [==============================] - 1s 147us/step - loss: 0.0393
Epoch 20/25
8000/8000 [==============================] - 1s 147us/step - loss: 0.0387
Epoch 21/25
8000/8000 [==============================] - 1s 147us/step - loss: 0.0382
Epoch 22/25
8000/8000 [==============================] - 1s 146us/step - loss: 0.0378
Epoch 23/25
8000/8000 [==============================] - 1s 148us/step - loss: 0.0376
Epoch 24/25
8000/8000 [==============================] - 1s 148us/step - loss: 0.0374
Epoch 25/25
8000/8000 [==============================] - 1s 148us/step - loss: 0.0373
Epoch 1/25
8000/8000 [==============================] - 2s 279us/step - loss: 0.0891
Epoch 2/25
8000/8000 [==============================] - 1s 173us/step - loss: 0.0592
Epoch 3/25
8000/8000 [==============================] - 1s 175us/step - loss: 0.0333
Epoch 4/25
8000/8000 [==============================] - 1s 176us/step - loss: 0.0222
Epoch 5/25
8000/8000 [==============================] - 1s 176us/step - loss: 0.0186
Epoch 6/25
8000/8000 [==============================] - 1s 175us/step - loss: 0.0164
Epoch 7/25
8000/8000 [==============================] - 1s 180us/step - loss: 0.0153
Epoch 8/25
8000/8000 [==============================] - 1s 172us/step - loss: 0.0145
Epoch 9/25
8000/8000 [==============================] - 1s 175us/step - loss: 0.0138
Epoch 10/25
8000/8000 [==============================] - 1s 170us/step - loss: 0.0132
Epoch 11/25
8000/8000 [==============================] - 1s 177us/step - loss: 0.0127
Epoch 12/25
8000/8000 [==============================] - 1s 177us/step - loss: 0.0121
Epoch 13/25
8000/8000 [==============================] - 1s 177us/step - loss: 0.0117
Epoch 14/25
8000/8000 [==============================] - 1s 174us/step - loss: 0.0113
Epoch 15/25
8000/8000 [==============================] - 1s 176us/step - loss: 0.0108
Epoch 16/25
8000/8000 [==============================] - 1s 173us/step - loss: 0.0104
Epoch 17/25
8000/8000 [==============================] - 1s 172us/step - loss: 0.0101
Epoch 18/25
8000/8000 [==============================] - 1s 177us/step - loss: 0.0098
Epoch 19/25
8000/8000 [==============================] - 1s 171us/step - loss: 0.0094
Epoch 20/25
8000/8000 [==============================] - 1s 174us/step - loss: 0.0091
Epoch 21/25
8000/8000 [==============================] - 1s 176us/step - loss: 0.0089
Epoch 22/25
8000/8000 [==============================] - 1s 175us/step - loss: 0.0086
Epoch 23/25
8000/8000 [==============================] - 1s 175us/step - loss: 0.0084
Epoch 24/25
8000/8000 [==============================] - 1s 171us/step - loss: 0.0081
Epoch 25/25
8000/8000 [==============================] - 1s 177us/step - loss: 0.0079
Epoch 1/25
8000/8000 [==============================] - 3s 324us/step - loss: 0.0947
Epoch 2/25
8000/8000 [==============================] - 2s 211us/step - loss: 0.0579
Epoch 3/25
8000/8000 [==============================] - 2s 214us/step - loss: 0.0260
Epoch 4/25
8000/8000 [==============================] - 2s 215us/step - loss: 0.0144
Epoch 5/25
8000/8000 [==============================] - 2s 217us/step - loss: 0.0110
Epoch 6/25
8000/8000 [==============================] - 2s 209us/step - loss: 0.0093
Epoch 7/25
8000/8000 [==============================] - 2s 210us/step - loss: 0.0081
Epoch 8/25
8000/8000 [==============================] - 2s 212us/step - loss: 0.0071
Epoch 9/25
8000/8000 [==============================] - 2s 211us/step - loss: 0.0063
Epoch 10/25
8000/8000 [==============================] - 2s 208us/step - loss: 0.0056
Epoch 11/25
8000/8000 [==============================] - 2s 206us/step - loss: 0.0050
Epoch 12/25
8000/8000 [==============================] - 2s 214us/step - loss: 0.0045
Epoch 13/25
8000/8000 [==============================] - 2s 217us/step - loss: 0.0041
Epoch 14/25
8000/8000 [==============================] - 2s 209us/step - loss: 0.0037
Epoch 15/25
8000/8000 [==============================] - 2s 214us/step - loss: 0.0033
Epoch 16/25
8000/8000 [==============================] - 2s 212us/step - loss: 0.0031
Epoch 17/25
8000/8000 [==============================] - 2s 211us/step - loss: 0.0028
Epoch 18/25
8000/8000 [==============================] - 2s 209us/step - loss: 0.0026
Epoch 19/25
8000/8000 [==============================] - 2s 208us/step - loss: 0.0025
Epoch 20/25
8000/8000 [==============================] - 2s 211us/step - loss: 0.0022
Epoch 21/25
8000/8000 [==============================] - 2s 208us/step - loss: 0.0021
Epoch 22/25
8000/8000 [==============================] - 2s 210us/step - loss: 0.0021
Epoch 23/25
8000/8000 [==============================] - 2s 207us/step - loss: 0.0019
Epoch 24/25
8000/8000 [==============================] - 2s 212us/step - loss: 0.0018
Epoch 25/25
8000/8000 [==============================] - 2s 211us/step - loss: 0.0018
mse ev: 0.000000; mse rr: 0.055404 (0.060183); mse simec (0hl): 0.055930 (0.062385); mse simec (2hl): 0.012982 (0.029643); mse simec (5hl): 0.002826 (0.026544)
15
Epoch 1/25
8000/8000 [==============================] - 2s 253us/step - loss: 0.1096
Epoch 2/25
8000/8000 [==============================] - 1s 154us/step - loss: 0.0886
Epoch 3/25
8000/8000 [==============================] - 1s 153us/step - loss: 0.0840
Epoch 4/25
8000/8000 [==============================] - 1s 155us/step - loss: 0.0818
Epoch 5/25
8000/8000 [==============================] - 1s 150us/step - loss: 0.0799
Epoch 6/25
8000/8000 [==============================] - 1s 149us/step - loss: 0.0778
Epoch 7/25
8000/8000 [==============================] - 1s 151us/step - loss: 0.0753
Epoch 8/25
8000/8000 [==============================] - 1s 153us/step - loss: 0.0720
Epoch 9/25
8000/8000 [==============================] - 1s 154us/step - loss: 0.0679
Epoch 10/25
8000/8000 [==============================] - 1s 146us/step - loss: 0.0634
Epoch 11/25
8000/8000 [==============================] - 1s 152us/step - loss: 0.0588
Epoch 12/25
8000/8000 [==============================] - 1s 147us/step - loss: 0.0545
Epoch 13/25
8000/8000 [==============================] - 1s 152us/step - loss: 0.0511
Epoch 14/25
8000/8000 [==============================] - 1s 156us/step - loss: 0.0486
Epoch 15/25
8000/8000 [==============================] - 1s 150us/step - loss: 0.0466
Epoch 16/25
8000/8000 [==============================] - 1s 155us/step - loss: 0.0450
Epoch 17/25
8000/8000 [==============================] - 1s 154us/step - loss: 0.0433
Epoch 18/25
8000/8000 [==============================] - 1s 152us/step - loss: 0.0416
Epoch 19/25
8000/8000 [==============================] - 1s 153us/step - loss: 0.0401
Epoch 20/25
8000/8000 [==============================] - 1s 151us/step - loss: 0.0390
Epoch 21/25
8000/8000 [==============================] - 1s 151us/step - loss: 0.0382
Epoch 22/25
8000/8000 [==============================] - 1s 154us/step - loss: 0.0378
Epoch 23/25
8000/8000 [==============================] - 1s 150us/step - loss: 0.0376
Epoch 24/25
8000/8000 [==============================] - 1s 153us/step - loss: 0.0374
Epoch 25/25
8000/8000 [==============================] - 1s 151us/step - loss: 0.0373
Epoch 1/25
8000/8000 [==============================] - 2s 298us/step - loss: 0.0858
Epoch 2/25
8000/8000 [==============================] - 1s 178us/step - loss: 0.0583
Epoch 3/25
8000/8000 [==============================] - 1s 182us/step - loss: 0.0377
Epoch 4/25
8000/8000 [==============================] - 1s 176us/step - loss: 0.0252
Epoch 5/25
8000/8000 [==============================] - 1s 181us/step - loss: 0.0198
Epoch 6/25
8000/8000 [==============================] - 1s 177us/step - loss: 0.0168
Epoch 7/25
8000/8000 [==============================] - 1s 178us/step - loss: 0.0154
Epoch 8/25
8000/8000 [==============================] - 1s 176us/step - loss: 0.0144
Epoch 9/25
8000/8000 [==============================] - 1s 177us/step - loss: 0.0136
Epoch 10/25
8000/8000 [==============================] - 1s 182us/step - loss: 0.0130
Epoch 11/25
8000/8000 [==============================] - 1s 180us/step - loss: 0.0124
Epoch 12/25
8000/8000 [==============================] - 1s 183us/step - loss: 0.0119
Epoch 13/25
8000/8000 [==============================] - 1s 179us/step - loss: 0.0113
Epoch 14/25
8000/8000 [==============================] - 1s 176us/step - loss: 0.0109
Epoch 15/25
8000/8000 [==============================] - 1s 179us/step - loss: 0.0105
Epoch 16/25
8000/8000 [==============================] - 1s 180us/step - loss: 0.0100
Epoch 17/25
8000/8000 [==============================] - 1s 178us/step - loss: 0.0097
Epoch 18/25
8000/8000 [==============================] - 1s 177us/step - loss: 0.0093
Epoch 19/25
8000/8000 [==============================] - 1s 175us/step - loss: 0.0090
Epoch 20/25
8000/8000 [==============================] - 1s 178us/step - loss: 0.0087
Epoch 21/25
8000/8000 [==============================] - 1s 176us/step - loss: 0.0084
Epoch 22/25
8000/8000 [==============================] - 1s 180us/step - loss: 0.0081
Epoch 23/25
8000/8000 [==============================] - 1s 178us/step - loss: 0.0079
Epoch 24/25
8000/8000 [==============================] - 1s 179us/step - loss: 0.0077
Epoch 25/25
8000/8000 [==============================] - 1s 179us/step - loss: 0.0075
Epoch 1/25
8000/8000 [==============================] - 3s 333us/step - loss: 0.0920
Epoch 2/25
8000/8000 [==============================] - 2s 214us/step - loss: 0.0562
Epoch 3/25
8000/8000 [==============================] - 2s 210us/step - loss: 0.0286
Epoch 4/25
8000/8000 [==============================] - 2s 211us/step - loss: 0.0145
Epoch 5/25
8000/8000 [==============================] - 2s 207us/step - loss: 0.0111
Epoch 6/25
8000/8000 [==============================] - 2s 209us/step - loss: 0.0095
Epoch 7/25
8000/8000 [==============================] - 2s 212us/step - loss: 0.0085
Epoch 8/25
8000/8000 [==============================] - 2s 216us/step - loss: 0.0075
Epoch 9/25
8000/8000 [==============================] - 2s 214us/step - loss: 0.0067
Epoch 10/25
8000/8000 [==============================] - 2s 216us/step - loss: 0.0060
Epoch 11/25
8000/8000 [==============================] - 2s 211us/step - loss: 0.0054
Epoch 12/25
8000/8000 [==============================] - 2s 214us/step - loss: 0.0048
Epoch 13/25
8000/8000 [==============================] - 2s 206us/step - loss: 0.0043
Epoch 14/25
8000/8000 [==============================] - 2s 212us/step - loss: 0.0039
Epoch 15/25
8000/8000 [==============================] - 2s 214us/step - loss: 0.0036
Epoch 16/25
8000/8000 [==============================] - 2s 212us/step - loss: 0.0032
Epoch 17/25
8000/8000 [==============================] - 2s 212us/step - loss: 0.0030
Epoch 18/25
8000/8000 [==============================] - 2s 208us/step - loss: 0.0027
Epoch 19/25
8000/8000 [==============================] - 2s 216us/step - loss: 0.0025
Epoch 20/25
8000/8000 [==============================] - 2s 212us/step - loss: 0.0024
Epoch 21/25
8000/8000 [==============================] - 2s 214us/step - loss: 0.0022
Epoch 22/25
8000/8000 [==============================] - 2s 213us/step - loss: 0.0021
Epoch 23/25
8000/8000 [==============================] - 2s 215us/step - loss: 0.0019
Epoch 24/25
8000/8000 [==============================] - 2s 214us/step - loss: 0.0018
Epoch 25/25
8000/8000 [==============================] - 2s 212us/step - loss: 0.0017
mse ev: 0.000000; mse rr: 0.055404 (0.060183); mse simec (0hl): 0.055744 (0.062352); mse simec (2hl): 0.012640 (0.030608); mse simec (5hl): 0.002708 (0.026025)
e_dims= [2, 3, 4, 5, 6, 7, 8, 9, 10, 15]
mse_ev= [0.066419542359582293, 0.055817708926609436, 0.045566146426609618, 0.03561653059674811, 0.025952881207718905, 0.016658888345945339, 0.0081564788795163434, 5.6042560830257886e-27, 5.604119942053966e-27, 5.6039813227498701e-27]
mse_rr= [0.077785515221909782, 0.072825996063333842, 0.068303733602481528, 0.064544647408199707, 0.062132954275257987, 0.059271531912452512, 0.057297454174229032, 0.055403880119418193, 0.055403880119418221, 0.055403880119418221]
mse_rr_test= [0.079355114740819022, 0.075058628769985855, 0.071993964074924865, 0.069153214970815557, 0.066742286814929733, 0.064193869503442441, 0.062026821418338915, 0.060183319509258877, 0.060183319509258933, 0.060183319509258926]
mse_simec= [0.076568046805416123, 0.07111271742053589, 0.067146942010457039, 0.063182462698062658, 0.060091321663356088, 0.058144739829429755, 0.056856916074838948, 0.05593356981806006, 0.055929581340072282, 0.05574435678751205]
mse_simec_test= [0.078447797453583809, 0.074487398805980223, 0.071882431234793348, 0.068509240286915601, 0.064931490262574998, 0.063480548424659375, 0.062826774230821697, 0.062521368328840018, 0.062384919434401019, 0.062351741685101554]
mse_simec_hl= [0.067426770052798499, 0.057223720744684223, 0.047863234395283401, 0.038889616745059405, 0.032779155864087403, 0.025566845067615415, 0.018793763760470032, 0.014201089718659637, 0.012981628118715187, 0.0126401737089]
mse_simec_hl_test= [0.072142132963918693, 0.064911174302152819, 0.058643928072935669, 0.051734722752925524, 0.047311863712150688, 0.041185981689874591, 0.036312581727810121, 0.031433373968742656, 0.029643204745305202, 0.0306086838695]
mse_simec_5hl= [0.067051845925757109, 0.056174430580778562, 0.046096776393505949, 0.036935119514275312, 0.027732720434574296, 0.019749833521688138, 0.011404728609891833, 0.0024818571207765655, 0.0028257408357321382, 0.002707684111031229]
mse_simec_5hl_test= [0.071175910865522843, 0.063413382177239408, 0.056709233686074535, 0.051086213440916566, 0.044222967741257842, 0.038145646971133304, 0.033130372561459738, 0.025713800840658954, 0.026543535924974403, 0.026025473719606205]

Embedding MNIST based on RBF kernel: embedding vs. prediction; number of targets; missing values



In [6]:

    
# Compute Gaussian kernel
D = squareform(pdist(X, 'euclidean'))
sigma = np.median(D)
gamma = 0.5/(sigma**2)
print("gamma: %.5f" % gamma)
K_rbf = center_K(rbf_kernel(X, X, gamma))
K_rbf_test = center_K(rbf_kernel(X_test, X_test, gamma))
K_rbf_test_train = center_K(rbf_kernel(np.concatenate([X,X_test]), np.concatenate([X,X_test]), gamma))[n_train:, :n_train]
# scale to be in the range [-1, 1]?
print(np.max(np.abs(K_rbf)))









    



gamma: 0.00475
0.735166002796



In [7]:

    
# check how many relevant dimensions there are
eigenvals = np.linalg.eigvalsh(K_rbf)[::-1]
plt.figure();
plt.plot(list(range(1, K_rbf.shape[0]+1)), eigenvals, '-o', markersize=3);
plt.plot([1, K_rbf.shape[0]],[0,0], 'k--', linewidth=0.5);
plt.xlim(1, 100);
plt.title('Eigenvalue Spectrum of the RBF Kernel');
# kernel PCA embedding
kpca = KernelPCA(n_components=2, kernel='rbf', gamma=gamma)
X_embed = kpca.fit_transform(X)
X_embed_test = kpca.transform(X_test)
plot_mnist(X_embed, y, X_embed_test, y_test, title='MNIST - RBF Kernel PCA')



In [8]:

    
e_dim = 10
mse_train, mse_test, mse_traintest, mse_pred_train, mse_pred_tt = [], [], [], [], []
s_ll_regs = [0., 0.0001, 0.001, 0.01, 0.1, 0.5, 1.]
for l in s_ll_regs:
    simec = SimilarityEncoder(X.shape[1], e_dim, K_rbf.shape[1], hidden_layers=[(1000, 'tanh')], 
                              l2_reg=0.00000001, l2_reg_emb=0.00001, l2_reg_out=0.0000001, 
                              s_ll_reg=l, S_ll=K_rbf, opt=keras.optimizers.Adamax(lr=0.0005))
    simec.fit(X, K_rbf)
    X_embeds = simec.transform(X)
    X_embed_tests = simec.transform(X_test)
    mse = check_similarity_match(X_embeds, K_rbf)[0]
    mse_t = check_similarity_match(X_embed_tests, K_rbf_test)[0]
    mse_tt = check_similarity_match(X_embed_tests.dot(X_embeds.T), K_rbf_test_train, X_embed_is_S_approx=True)[0]
    mse_p = check_similarity_match(simec.predict(X), K_rbf, X_embed_is_S_approx=True)[0]
    mse_pt = check_similarity_match(simec.predict(X_test), K_rbf_test_train, X_embed_is_S_approx=True)[0]
    mse_train.append(mse)
    mse_test.append(mse_t)
    mse_traintest.append(mse_tt)
    mse_pred_train.append(mse_p)
    mse_pred_tt.append(mse_pt)
    print("embedding: %.5f (%.5f; %.5f); prediction: %.5f (%.5f)" % (mse, mse_tt, mse_t, mse_p, mse_pt))
keras.backend.clear_session()
print("s_ll_regs=", s_ll_regs)
print("mse_train=", mse_train)
print("mse_traintest=", mse_traintest)
print("mse_test=", mse_test)
print("mse_pred_train=", mse_pred_train)
print("mse_pred_tt=", mse_pred_tt)
colors = get_colors(15)
plt.figure();
plt.plot(s_ll_regs, mse_train, '-o', markersize=3, c=colors[0], label='$YY^{\\top}$ train');
plt.plot(s_ll_regs, mse_traintest, '-o', markersize=3, c=colors[2], label='$YY^{\\top}$ train/test');
plt.plot(s_ll_regs, mse_test, '-o', markersize=3, c=colors[4], label='$YY^{\\top}$ test');
plt.plot(s_ll_regs, mse_pred_train, '-o', markersize=3, c=colors[8], label='$YW_l$ train');
plt.plot(s_ll_regs, mse_pred_tt, '-o', markersize=3, c=colors[9], label='$YW_l$ train/test');
plt.legend(loc=0);
plt.title('MNIST (RBF kernel)');
plt.plot([0, s_ll_regs[-1]], [0,0], 'k--', linewidth=0.5);
plt.xticks([0., 0.1, 0.5, 1.], [0., 0.1, 0.5, 1.]);
plt.xlabel('Regularization strength ($\lambda$)')
plt.ylabel('Mean Squared Error of $\hat{S}$')
if savefigs: plt.savefig('fig_rbf_mse_sllreg.pdf', dpi=300, bbox_inches="tight")









    



Epoch 1/25
8000/8000 [==============================] - 1s 154us/step - loss: 0.0017
Epoch 2/25
8000/8000 [==============================] - 1s 135us/step - loss: 7.2949e-04
Epoch 3/25
8000/8000 [==============================] - 1s 135us/step - loss: 5.5195e-04
Epoch 4/25
8000/8000 [==============================] - 1s 138us/step - loss: 4.8630e-04
Epoch 5/25
8000/8000 [==============================] - 1s 133us/step - loss: 4.6890e-04
Epoch 6/25
8000/8000 [==============================] - 1s 131us/step - loss: 4.6224e-04
Epoch 7/25
8000/8000 [==============================] - 1s 133us/step - loss: 4.5897e-04
Epoch 8/25
8000/8000 [==============================] - 1s 139us/step - loss: 4.5687e-04
Epoch 9/25
8000/8000 [==============================] - 1s 130us/step - loss: 4.5536e-04
Epoch 10/25
8000/8000 [==============================] - 1s 131us/step - loss: 4.5449e-04
Epoch 11/25
8000/8000 [==============================] - 1s 135us/step - loss: 4.5349e-04
Epoch 12/25
8000/8000 [==============================] - 1s 138us/step - loss: 4.5288e-04
Epoch 13/25
8000/8000 [==============================] - 1s 132us/step - loss: 4.5230e-04
Epoch 14/25
8000/8000 [==============================] - 1s 133us/step - loss: 4.5188e-04
Epoch 15/25
8000/8000 [==============================] - 1s 138us/step - loss: 4.5136e-04
Epoch 16/25
8000/8000 [==============================] - 1s 138us/step - loss: 4.5094e-04
Epoch 17/25
8000/8000 [==============================] - 1s 133us/step - loss: 4.5056e-04
Epoch 18/25
8000/8000 [==============================] - 1s 136us/step - loss: 4.5028e-04
Epoch 19/25
8000/8000 [==============================] - 1s 134us/step - loss: 4.4991e-04
Epoch 20/25
8000/8000 [==============================] - 1s 131us/step - loss: 4.4985e-04
Epoch 21/25
8000/8000 [==============================] - 1s 136us/step - loss: 4.4951e-04
Epoch 22/25
8000/8000 [==============================] - 1s 137us/step - loss: 4.4913e-04
Epoch 23/25
8000/8000 [==============================] - 1s 138us/step - loss: 4.4885e-04
Epoch 24/25
8000/8000 [==============================] - 1s 134us/step - loss: 4.4859e-04
Epoch 25/25
8000/8000 [==============================] - 1s 137us/step - loss: 4.4857e-04
embedding: 4.47659 (4.38161; 4.32224); prediction: 0.00044 (0.00043)
Epoch 1/25
8000/8000 [==============================] - 11s 1ms/step - loss: 0.0017
Epoch 2/25
8000/8000 [==============================] - 8s 951us/step - loss: 7.2897e-04
Epoch 3/25
8000/8000 [==============================] - 8s 950us/step - loss: 5.5146e-04
Epoch 4/25
8000/8000 [==============================] - 8s 952us/step - loss: 4.9202e-04
Epoch 5/25
8000/8000 [==============================] - 8s 952us/step - loss: 4.7237e-04
Epoch 6/25
8000/8000 [==============================] - 8s 949us/step - loss: 4.6473e-04
Epoch 7/25
8000/8000 [==============================] - 8s 951us/step - loss: 4.6038e-04
Epoch 8/25
8000/8000 [==============================] - 8s 952us/step - loss: 4.5794e-04
Epoch 9/25
8000/8000 [==============================] - 8s 953us/step - loss: 4.5604e-04
Epoch 10/25
8000/8000 [==============================] - 8s 953us/step - loss: 4.5495e-04
Epoch 11/25
8000/8000 [==============================] - 8s 952us/step - loss: 4.5398e-04
Epoch 12/25
8000/8000 [==============================] - 8s 950us/step - loss: 4.5317e-04
Epoch 13/25
8000/8000 [==============================] - 8s 950us/step - loss: 4.5274e-04
Epoch 14/25
8000/8000 [==============================] - 8s 951us/step - loss: 4.5203e-04
Epoch 15/25
8000/8000 [==============================] - 8s 950us/step - loss: 4.5158e-04
Epoch 16/25
8000/8000 [==============================] - 8s 951us/step - loss: 4.5138e-04
Epoch 17/25
8000/8000 [==============================] - 8s 949us/step - loss: 4.5086e-04
Epoch 18/25
8000/8000 [==============================] - 8s 949us/step - loss: 4.5084e-04
Epoch 19/25
8000/8000 [==============================] - 8s 951us/step - loss: 4.5011e-04
Epoch 20/25
8000/8000 [==============================] - 8s 949us/step - loss: 4.5016e-04
Epoch 21/25
8000/8000 [==============================] - 8s 946us/step - loss: 4.4989e-04
Epoch 22/25
8000/8000 [==============================] - 8s 985us/step - loss: 4.4966e-04
Epoch 23/25
8000/8000 [==============================] - 8s 960us/step - loss: 4.4940e-04
Epoch 24/25
8000/8000 [==============================] - 8s 959us/step - loss: 4.4893e-04
Epoch 25/25
8000/8000 [==============================] - 8s 955us/step - loss: 4.4891e-04
embedding: 4.34288 (4.24484; 4.18137); prediction: 0.00044 (0.00043)
Epoch 1/25
8000/8000 [==============================] - 11s 1ms/step - loss: 0.0017
Epoch 2/25
8000/8000 [==============================] - 8s 948us/step - loss: 7.2773e-04
Epoch 3/25
8000/8000 [==============================] - 8s 948us/step - loss: 5.4502e-04
Epoch 4/25
8000/8000 [==============================] - 8s 949us/step - loss: 4.9266e-04
Epoch 5/25
8000/8000 [==============================] - 8s 944us/step - loss: 4.7612e-04
Epoch 6/25
8000/8000 [==============================] - 8s 944us/step - loss: 4.6839e-04
Epoch 7/25
8000/8000 [==============================] - 8s 945us/step - loss: 4.6424e-04
Epoch 8/25
8000/8000 [==============================] - 8s 952us/step - loss: 4.6108e-04
Epoch 9/25
8000/8000 [==============================] - 8s 948us/step - loss: 4.5912e-04
Epoch 10/25
8000/8000 [==============================] - 8s 952us/step - loss: 4.5788e-04
Epoch 11/25
8000/8000 [==============================] - 8s 951us/step - loss: 4.5705e-04
Epoch 12/25
8000/8000 [==============================] - 8s 949us/step - loss: 4.5594e-04
Epoch 13/25
8000/8000 [==============================] - 8s 949us/step - loss: 4.5561e-04
Epoch 14/25
8000/8000 [==============================] - 8s 946us/step - loss: 4.5495e-04
Epoch 15/25
8000/8000 [==============================] - 8s 941us/step - loss: 4.5443e-04
Epoch 16/25
8000/8000 [==============================] - 8s 946us/step - loss: 4.5404e-04
Epoch 17/25
8000/8000 [==============================] - 8s 948us/step - loss: 4.5375e-04
Epoch 18/25
8000/8000 [==============================] - 8s 950us/step - loss: 4.5340e-04
Epoch 19/25
8000/8000 [==============================] - 8s 949us/step - loss: 4.5310e-04
Epoch 20/25
8000/8000 [==============================] - 8s 951us/step - loss: 4.5262e-04
Epoch 21/25
8000/8000 [==============================] - 8s 947us/step - loss: 4.5271e-04
Epoch 22/25
8000/8000 [==============================] - 8s 947us/step - loss: 4.5224e-04
Epoch 23/25
8000/8000 [==============================] - 8s 943us/step - loss: 4.5210e-04
Epoch 24/25
8000/8000 [==============================] - 8s 946us/step - loss: 4.5195e-04
Epoch 25/25
8000/8000 [==============================] - 8s 945us/step - loss: 4.5175e-04
embedding: 4.27038 (4.18080; 4.12478); prediction: 0.00043 (0.00043)
Epoch 1/25
8000/8000 [==============================] - 11s 1ms/step - loss: 0.0017
Epoch 2/25
8000/8000 [==============================] - 8s 952us/step - loss: 7.5504e-04
Epoch 3/25
8000/8000 [==============================] - 8s 953us/step - loss: 5.6621e-04
Epoch 4/25
8000/8000 [==============================] - 8s 951us/step - loss: 5.1741e-04
Epoch 5/25
8000/8000 [==============================] - 8s 944us/step - loss: 5.0256e-04
Epoch 6/25
8000/8000 [==============================] - 8s 950us/step - loss: 4.9516e-04
Epoch 7/25
8000/8000 [==============================] - 8s 958us/step - loss: 4.9118e-04
Epoch 8/25
8000/8000 [==============================] - 8s 957us/step - loss: 4.8850e-04
Epoch 9/25
8000/8000 [==============================] - 8s 951us/step - loss: 4.8698e-04
Epoch 10/25
8000/8000 [==============================] - 8s 952us/step - loss: 4.8549e-04
Epoch 11/25
8000/8000 [==============================] - 8s 952us/step - loss: 4.8471e-04
Epoch 12/25
8000/8000 [==============================] - 8s 956us/step - loss: 4.8387e-04
Epoch 13/25
8000/8000 [==============================] - 8s 954us/step - loss: 4.8338e-04
Epoch 14/25
8000/8000 [==============================] - 8s 952us/step - loss: 4.8287e-04
Epoch 15/25
8000/8000 [==============================] - 8s 954us/step - loss: 4.8237e-04
Epoch 16/25
8000/8000 [==============================] - 8s 954us/step - loss: 4.8218e-04
Epoch 17/25
8000/8000 [==============================] - 8s 953us/step - loss: 4.8147e-04
Epoch 18/25
8000/8000 [==============================] - 8s 952us/step - loss: 4.8120e-04
Epoch 19/25
8000/8000 [==============================] - 8s 954us/step - loss: 4.8086e-04
Epoch 20/25
8000/8000 [==============================] - 8s 957us/step - loss: 4.8070e-04
Epoch 21/25
8000/8000 [==============================] - 8s 955us/step - loss: 4.8030e-04
Epoch 22/25
8000/8000 [==============================] - 8s 956us/step - loss: 4.8012e-04
Epoch 23/25
8000/8000 [==============================] - 8s 956us/step - loss: 4.7992e-04
Epoch 24/25
8000/8000 [==============================] - 8s 949us/step - loss: 4.7972e-04
Epoch 25/25
8000/8000 [==============================] - 8s 958us/step - loss: 4.7943e-04
embedding: 3.28897 (3.21086; 3.15931); prediction: 0.00043 (0.00043)
Epoch 1/25
8000/8000 [==============================] - 11s 1ms/step - loss: 0.0020
Epoch 2/25
8000/8000 [==============================] - 8s 955us/step - loss: 0.0011
Epoch 3/25
8000/8000 [==============================] - 8s 955us/step - loss: 8.6965e-04
Epoch 4/25
8000/8000 [==============================] - 8s 952us/step - loss: 7.9976e-04
Epoch 5/25
8000/8000 [==============================] - 8s 952us/step - loss: 7.8257e-04
Epoch 6/25
8000/8000 [==============================] - 8s 958us/step - loss: 7.7450e-04
Epoch 7/25
8000/8000 [==============================] - 8s 954us/step - loss: 7.6952e-04
Epoch 8/25
8000/8000 [==============================] - 8s 957us/step - loss: 7.6585e-04
Epoch 9/25
8000/8000 [==============================] - 8s 955us/step - loss: 7.6288e-04
Epoch 10/25
8000/8000 [==============================] - 8s 954us/step - loss: 7.6000e-04
Epoch 11/25
8000/8000 [==============================] - 8s 952us/step - loss: 7.5727e-04
Epoch 12/25
8000/8000 [==============================] - 8s 952us/step - loss: 7.5417e-04
Epoch 13/25
8000/8000 [==============================] - 8s 955us/step - loss: 7.5110e-04
Epoch 14/25
8000/8000 [==============================] - 8s 955us/step - loss: 7.4751e-04
Epoch 15/25
8000/8000 [==============================] - 8s 955us/step - loss: 7.4358e-04
Epoch 16/25
8000/8000 [==============================] - 8s 957us/step - loss: 7.3912e-04
Epoch 17/25
8000/8000 [==============================] - 8s 956us/step - loss: 7.3390e-04
Epoch 18/25
8000/8000 [==============================] - 8s 954us/step - loss: 7.2810e-04
Epoch 19/25
8000/8000 [==============================] - 8s 954us/step - loss: 7.2139e-04
Epoch 20/25
8000/8000 [==============================] - 8s 950us/step - loss: 7.1416e-04
Epoch 21/25
8000/8000 [==============================] - 8s 954us/step - loss: 7.0614e-04
Epoch 22/25
8000/8000 [==============================] - 8s 953us/step - loss: 6.9689e-04
Epoch 23/25
8000/8000 [==============================] - 8s 958us/step - loss: 6.8800e-04
Epoch 24/25
8000/8000 [==============================] - 8s 953us/step - loss: 6.7955e-04
Epoch 25/25
8000/8000 [==============================] - 8s 947us/step - loss: 6.7092e-04
embedding: 0.14083 (0.13620; 0.13254); prediction: 0.00044 (0.00043)
Epoch 1/25
8000/8000 [==============================] - 11s 1ms/step - loss: 0.0033
Epoch 2/25
8000/8000 [==============================] - 8s 955us/step - loss: 0.0023
Epoch 3/25
8000/8000 [==============================] - 8s 954us/step - loss: 0.0021
Epoch 4/25
8000/8000 [==============================] - 8s 952us/step - loss: 0.0020
Epoch 5/25
8000/8000 [==============================] - 8s 950us/step - loss: 0.0019
Epoch 6/25
8000/8000 [==============================] - 8s 948us/step - loss: 0.0018
Epoch 7/25
8000/8000 [==============================] - 8s 947us/step - loss: 0.0017
Epoch 8/25
8000/8000 [==============================] - 8s 948us/step - loss: 0.0014
Epoch 9/25
8000/8000 [==============================] - 8s 950us/step - loss: 0.0012
Epoch 10/25
8000/8000 [==============================] - 8s 948us/step - loss: 0.0010
Epoch 11/25
8000/8000 [==============================] - 8s 952us/step - loss: 9.2182e-04
Epoch 12/25
8000/8000 [==============================] - 8s 952us/step - loss: 8.5703e-04
Epoch 13/25
8000/8000 [==============================] - 8s 953us/step - loss: 8.2008e-04
Epoch 14/25
8000/8000 [==============================] - 8s 951us/step - loss: 7.9784e-04
Epoch 15/25
8000/8000 [==============================] - 8s 954us/step - loss: 7.8601e-04
Epoch 16/25
8000/8000 [==============================] - 8s 950us/step - loss: 7.8104e-04
Epoch 17/25
8000/8000 [==============================] - 8s 949us/step - loss: 7.8005e-04
Epoch 18/25
8000/8000 [==============================] - 8s 955us/step - loss: 7.7929e-04
Epoch 19/25
8000/8000 [==============================] - 8s 983us/step - loss: 7.7875e-04
Epoch 20/25
8000/8000 [==============================] - 8s 971us/step - loss: 7.7864e-04
Epoch 21/25
8000/8000 [==============================] - 8s 955us/step - loss: 7.7797e-04
Epoch 22/25
8000/8000 [==============================] - 8s 953us/step - loss: 7.7795e-04
Epoch 23/25
8000/8000 [==============================] - 8s 954us/step - loss: 7.7716e-04
Epoch 24/25
8000/8000 [==============================] - 8s 954us/step - loss: 7.7703e-04
Epoch 25/25
8000/8000 [==============================] - 8s 953us/step - loss: 7.7729e-04
embedding: 0.00045 (0.00044; 0.00048); prediction: 0.00044 (0.00043)
Epoch 1/25
8000/8000 [==============================] - 11s 1ms/step - loss: 0.0049
Epoch 2/25
8000/8000 [==============================] - 8s 953us/step - loss: 0.0039
Epoch 3/25
8000/8000 [==============================] - 8s 948us/step - loss: 0.0036
Epoch 4/25
8000/8000 [==============================] - 8s 954us/step - loss: 0.0033
Epoch 5/25
8000/8000 [==============================] - 8s 953us/step - loss: 0.0028
Epoch 6/25
8000/8000 [==============================] - 8s 952us/step - loss: 0.0020
Epoch 7/25
8000/8000 [==============================] - 8s 951us/step - loss: 0.0015
Epoch 8/25
8000/8000 [==============================] - 8s 951us/step - loss: 0.0012
Epoch 9/25
8000/8000 [==============================] - 8s 949us/step - loss: 0.0011
Epoch 10/25
8000/8000 [==============================] - 8s 954us/step - loss: 0.0010
Epoch 11/25
8000/8000 [==============================] - 8s 952us/step - loss: 0.0010
Epoch 12/25
8000/8000 [==============================] - 8s 951us/step - loss: 0.0010
Epoch 13/25
8000/8000 [==============================] - 8s 952us/step - loss: 0.0010
Epoch 14/25
8000/8000 [==============================] - 8s 947us/step - loss: 0.0010
Epoch 15/25
8000/8000 [==============================] - 8s 954us/step - loss: 0.0010
Epoch 16/25
8000/8000 [==============================] - 8s 952us/step - loss: 0.0010
Epoch 17/25
8000/8000 [==============================] - 8s 951us/step - loss: 0.0010
Epoch 18/25
8000/8000 [==============================] - 8s 949us/step - loss: 0.0010
Epoch 19/25
8000/8000 [==============================] - 8s 951us/step - loss: 9.9959e-04
Epoch 20/25
8000/8000 [==============================] - 8s 954us/step - loss: 9.9838e-04
Epoch 21/25
8000/8000 [==============================] - 8s 951us/step - loss: 9.9795e-04
Epoch 22/25
8000/8000 [==============================] - 8s 951us/step - loss: 9.9721e-04
Epoch 23/25
8000/8000 [==============================] - 8s 952us/step - loss: 9.9728e-04
Epoch 24/25
8000/8000 [==============================] - 8s 955us/step - loss: 9.9623e-04
Epoch 25/25
8000/8000 [==============================] - 8s 953us/step - loss: 9.9622e-04
embedding: 0.00044 (0.00043; 0.00047); prediction: 0.00044 (0.00043)
s_ll_regs= [0.0, 0.0001, 0.001, 0.01, 0.1, 0.5, 1.0]
mse_train= [4.4765935743321625, 4.3428841850874198, 4.2703824995185995, 3.2889747201907666, 0.14083360667558414, 0.00045114784967675039, 0.00044224526868058621]
mse_traintest= [4.3816142942458098, 4.2448434049733805, 4.1807974724847847, 3.2108622076036188, 0.13619886066665887, 0.00044194264887446221, 0.00043332245268069077]
mse_test= [4.3222385925647622, 4.1813698758339406, 4.124784061210736, 3.1593093138762436, 0.13254243778661373, 0.0004752773669672363, 0.00046951750803165767]
mse_pred_train= [0.00043512330567040721, 0.00043533578672172424, 0.0004348636335450985, 0.00043498600766803325, 0.0004376264267115522, 0.00043575607753341906, 0.00043629280990508279]
mse_pred_tt= [0.00042644535120334376, 0.00042724333383817358, 0.00042600831481940667, 0.00042642462379250805, 0.00042809171664935284, 0.00042682245839818517, 0.0004273433811002018]



In [9]:

    
# check effect of different number of targets
e_dim = 10
mse_simec, mse_simec_test = [], []
targets = [100, 250, 500, 750, 1000, 1500, 2500, 5000, n_train]
kpca = KernelPCA(n_components=e_dim, kernel='rbf', gamma=gamma)
X_embed = kpca.fit_transform(X)
X_embed_test = kpca.transform(X_test)
mse_k = check_similarity_match(X_embed, K_rbf)[0]
mse_kt = check_similarity_match(X_embed_test, K_rbf_test)[0]
for n in targets:
    print(n)
    simec = SimilarityEncoder(X.shape[1], e_dim, n, hidden_layers=[(1000, 'tanh')], 
                              l2_reg=0.00000001, l2_reg_emb=0.00001, l2_reg_out=0.0000001, 
                              s_ll_reg=5., S_ll=K_rbf[:n,:n], opt=keras.optimizers.Adamax(lr=0.0005))
    simec.fit(X, K_rbf[:,:n])
    X_embed = simec.transform(X)
    X_embed_test = simec.transform(X_test)
    mse = check_similarity_match(X_embed, K_rbf)[0]
    mse_simec.append(mse)
    mse_t = check_similarity_match(X_embed_test, K_rbf_test)[0]
    mse_simec_test.append(mse_t)
    print("mse kpca: %f (%f); mse simec: %f (%f)" % (mse_k, mse_kt, mse, mse_t))
keras.backend.clear_session()
print("targets=", targets)
print("mse_k=", mse_k)
print("mse_kt=", mse_kt)
print("mse_simec=", mse_simec)
print("mse_simec_test=", mse_simec_test)
colors = get_colors(10)
plt.figure();
plt.plot([0, targets[-1]], [mse_k, mse_k], '--', linewidth=0.5, c=colors[8], label='kPCA');
plt.plot([0, targets[-1]], [mse_kt, mse_kt], '--', linewidth=0.5, c=colors[6], label='kPCA (test)');
plt.plot(targets, mse_simec, '-o', markersize=3, c=colors[8], label='SimEc');
plt.plot(targets, mse_simec_test, '-o', markersize=3, c=colors[6], label='SimEc (test)');
plt.legend(loc=0);
plt.title('MNIST (RBF kernel)');
plt.xticks([100, 1000, 2500, 5000, 8000], [100, 1000, 2500, 5000, 8000]);
plt.xlabel('Number of Targets ($n$)')
plt.ylabel('Mean Squared Error of $\hat{S}$')
if savefigs: plt.savefig('fig_rbf_mse_ntargets.pdf', dpi=300, bbox_inches="tight")









    



100
Epoch 1/25
8000/8000 [==============================] - 1s 143us/step - loss: 0.0308
Epoch 2/25
8000/8000 [==============================] - 1s 121us/step - loss: 0.0202
Epoch 3/25
8000/8000 [==============================] - 1s 120us/step - loss: 0.0117
Epoch 4/25
8000/8000 [==============================] - 1s 118us/step - loss: 0.0077
Epoch 5/25
8000/8000 [==============================] - 1s 124us/step - loss: 0.0062
Epoch 6/25
8000/8000 [==============================] - 1s 117us/step - loss: 0.0055
Epoch 7/25
8000/8000 [==============================] - 1s 117us/step - loss: 0.0053
Epoch 8/25
8000/8000 [==============================] - 1s 121us/step - loss: 0.0052
Epoch 9/25
8000/8000 [==============================] - 1s 119us/step - loss: 0.0051
Epoch 10/25
8000/8000 [==============================] - 1s 114us/step - loss: 0.0050
Epoch 11/25
8000/8000 [==============================] - 1s 119us/step - loss: 0.0050
Epoch 12/25
8000/8000 [==============================] - 1s 117us/step - loss: 0.0050
Epoch 13/25
8000/8000 [==============================] - 1s 117us/step - loss: 0.0049
Epoch 14/25
8000/8000 [==============================] - 1s 122us/step - loss: 0.0049
Epoch 15/25
8000/8000 [==============================] - 1s 119us/step - loss: 0.0049
Epoch 16/25
8000/8000 [==============================] - 1s 113us/step - loss: 0.0049
Epoch 17/25
8000/8000 [==============================] - 1s 118us/step - loss: 0.0049
Epoch 18/25
8000/8000 [==============================] - 1s 117us/step - loss: 0.0049
Epoch 19/25
8000/8000 [==============================] - 1s 116us/step - loss: 0.0049
Epoch 20/25
8000/8000 [==============================] - 1s 119us/step - loss: 0.0049
Epoch 21/25
8000/8000 [==============================] - 1s 118us/step - loss: 0.0049
Epoch 22/25
8000/8000 [==============================] - 1s 116us/step - loss: 0.0048
Epoch 23/25
8000/8000 [==============================] - 1s 117us/step - loss: 0.0048
Epoch 24/25
8000/8000 [==============================] - 1s 117us/step - loss: 0.0048
Epoch 25/25
8000/8000 [==============================] - 1s 116us/step - loss: 0.0048
mse kpca: 0.000432 (0.000460); mse simec: 0.000650 (0.000691)
250
Epoch 1/25
8000/8000 [==============================] - 1s 138us/step - loss: 0.0208
Epoch 2/25
8000/8000 [==============================] - 1s 120us/step - loss: 0.0114
Epoch 3/25
8000/8000 [==============================] - 1s 118us/step - loss: 0.0062
Epoch 4/25
8000/8000 [==============================] - 1s 122us/step - loss: 0.0048
Epoch 5/25
8000/8000 [==============================] - 1s 120us/step - loss: 0.0042
Epoch 6/25
8000/8000 [==============================] - 1s 117us/step - loss: 0.0040
Epoch 7/25
8000/8000 [==============================] - 1s 122us/step - loss: 0.0038
Epoch 8/25
8000/8000 [==============================] - 1s 120us/step - loss: 0.0037
Epoch 9/25
8000/8000 [==============================] - 1s 122us/step - loss: 0.0037
Epoch 10/25
8000/8000 [==============================] - 1s 122us/step - loss: 0.0037
Epoch 11/25
8000/8000 [==============================] - 1s 113us/step - loss: 0.0036
Epoch 12/25
8000/8000 [==============================] - 1s 120us/step - loss: 0.0036
Epoch 13/25
8000/8000 [==============================] - 1s 121us/step - loss: 0.0036
Epoch 14/25
8000/8000 [==============================] - 1s 119us/step - loss: 0.0036
Epoch 15/25
8000/8000 [==============================] - 1s 120us/step - loss: 0.0036
Epoch 16/25
8000/8000 [==============================] - 1s 114us/step - loss: 0.0036
Epoch 17/25
8000/8000 [==============================] - 1s 117us/step - loss: 0.0036
Epoch 18/25
8000/8000 [==============================] - 1s 120us/step - loss: 0.0036
Epoch 19/25
8000/8000 [==============================] - 1s 118us/step - loss: 0.0036
Epoch 20/25
8000/8000 [==============================] - 1s 118us/step - loss: 0.0035
Epoch 21/25
8000/8000 [==============================] - 1s 115us/step - loss: 0.0035
Epoch 22/25
8000/8000 [==============================] - 1s 121us/step - loss: 0.0035
Epoch 23/25
8000/8000 [==============================] - 1s 117us/step - loss: 0.0035
Epoch 24/25
8000/8000 [==============================] - 1s 117us/step - loss: 0.0035
Epoch 25/25
8000/8000 [==============================] - 1s 119us/step - loss: 0.0035
mse kpca: 0.000432 (0.000460); mse simec: 0.000551 (0.000580)
500
Epoch 1/25
8000/8000 [==============================] - 1s 142us/step - loss: 0.0191
Epoch 2/25
8000/8000 [==============================] - 1s 123us/step - loss: 0.0096
Epoch 3/25
8000/8000 [==============================] - 1s 119us/step - loss: 0.0052
Epoch 4/25
8000/8000 [==============================] - 1s 118us/step - loss: 0.0039
Epoch 5/25
8000/8000 [==============================] - 1s 119us/step - loss: 0.0035
Epoch 6/25
8000/8000 [==============================] - 1s 121us/step - loss: 0.0034
Epoch 7/25
8000/8000 [==============================] - 1s 119us/step - loss: 0.0033
Epoch 8/25
8000/8000 [==============================] - 1s 117us/step - loss: 0.0032
Epoch 9/25
8000/8000 [==============================] - 1s 121us/step - loss: 0.0032
Epoch 10/25
8000/8000 [==============================] - 1s 121us/step - loss: 0.0031
Epoch 11/25
8000/8000 [==============================] - 1s 125us/step - loss: 0.0031
Epoch 12/25
8000/8000 [==============================] - 1s 121us/step - loss: 0.0031
Epoch 13/25
8000/8000 [==============================] - 1s 116us/step - loss: 0.0031
Epoch 14/25
8000/8000 [==============================] - 1s 117us/step - loss: 0.0031
Epoch 15/25
8000/8000 [==============================] - 1s 118us/step - loss: 0.0031
Epoch 16/25
8000/8000 [==============================] - 1s 119us/step - loss: 0.0031
Epoch 17/25
8000/8000 [==============================] - 1s 122us/step - loss: 0.0031
Epoch 18/25
8000/8000 [==============================] - 1s 120us/step - loss: 0.0030
Epoch 19/25
8000/8000 [==============================] - 1s 115us/step - loss: 0.0030
Epoch 20/25
8000/8000 [==============================] - 1s 120us/step - loss: 0.0030
Epoch 21/25
8000/8000 [==============================] - 1s 118us/step - loss: 0.0030
Epoch 22/25
8000/8000 [==============================] - 1s 118us/step - loss: 0.0030
Epoch 23/25
8000/8000 [==============================] - 1s 121us/step - loss: 0.0030
Epoch 24/25
8000/8000 [==============================] - 1s 122us/step - loss: 0.0030
Epoch 25/25
8000/8000 [==============================] - 1s 120us/step - loss: 0.0030
mse kpca: 0.000432 (0.000460); mse simec: 0.000496 (0.000522)
750
Epoch 1/25
8000/8000 [==============================] - 1s 150us/step - loss: 0.0183
Epoch 2/25
8000/8000 [==============================] - 1s 116us/step - loss: 0.0090
Epoch 3/25
8000/8000 [==============================] - 1s 122us/step - loss: 0.0049
Epoch 4/25
8000/8000 [==============================] - 1s 120us/step - loss: 0.0037
Epoch 5/25
8000/8000 [==============================] - 1s 123us/step - loss: 0.0032
Epoch 6/25
8000/8000 [==============================] - 1s 124us/step - loss: 0.0031
Epoch 7/25
8000/8000 [==============================] - 1s 122us/step - loss: 0.0031
Epoch 8/25
8000/8000 [==============================] - 1s 119us/step - loss: 0.0031
Epoch 9/25
8000/8000 [==============================] - 1s 122us/step - loss: 0.0030
Epoch 10/25
8000/8000 [==============================] - 1s 121us/step - loss: 0.0030
Epoch 11/25
8000/8000 [==============================] - 1s 123us/step - loss: 0.0030
Epoch 12/25
8000/8000 [==============================] - 1s 117us/step - loss: 0.0030
Epoch 13/25
8000/8000 [==============================] - 1s 122us/step - loss: 0.0030
Epoch 14/25
8000/8000 [==============================] - 1s 121us/step - loss: 0.0030
Epoch 15/25
8000/8000 [==============================] - 1s 118us/step - loss: 0.0030
Epoch 16/25
8000/8000 [==============================] - 1s 114us/step - loss: 0.0030
Epoch 17/25
8000/8000 [==============================] - 1s 115us/step - loss: 0.0030
Epoch 18/25
8000/8000 [==============================] - 1s 122us/step - loss: 0.0029
Epoch 19/25
8000/8000 [==============================] - 1s 123us/step - loss: 0.0029
Epoch 20/25
8000/8000 [==============================] - 1s 127us/step - loss: 0.0029
Epoch 21/25
8000/8000 [==============================] - 1s 124us/step - loss: 0.0029
Epoch 22/25
8000/8000 [==============================] - 1s 119us/step - loss: 0.0029
Epoch 23/25
8000/8000 [==============================] - 1s 126us/step - loss: 0.0029
Epoch 24/25
8000/8000 [==============================] - 1s 123us/step - loss: 0.0029
Epoch 25/25
8000/8000 [==============================] - 1s 125us/step - loss: 0.0029
mse kpca: 0.000432 (0.000460); mse simec: 0.000485 (0.000512)
1000
Epoch 1/25
8000/8000 [==============================] - 1s 157us/step - loss: 0.0178
Epoch 2/25
8000/8000 [==============================] - 1s 129us/step - loss: 0.0087
Epoch 3/25
8000/8000 [==============================] - 1s 123us/step - loss: 0.0047
Epoch 4/25
8000/8000 [==============================] - 1s 124us/step - loss: 0.0035
Epoch 5/25
8000/8000 [==============================] - 1s 122us/step - loss: 0.0032
Epoch 6/25
8000/8000 [==============================] - 1s 126us/step - loss: 0.0030
Epoch 7/25
8000/8000 [==============================] - 1s 128us/step - loss: 0.0030
Epoch 8/25
8000/8000 [==============================] - 1s 130us/step - loss: 0.0030
Epoch 9/25
8000/8000 [==============================] - 1s 125us/step - loss: 0.0030
Epoch 10/25
8000/8000 [==============================] - 1s 127us/step - loss: 0.0030
Epoch 11/25
8000/8000 [==============================] - 1s 126us/step - loss: 0.0029
Epoch 12/25
8000/8000 [==============================] - 1s 127us/step - loss: 0.0029
Epoch 13/25
8000/8000 [==============================] - 1s 127us/step - loss: 0.0029
Epoch 14/25
8000/8000 [==============================] - 1s 127us/step - loss: 0.0029
Epoch 15/25
8000/8000 [==============================] - 1s 125us/step - loss: 0.0029
Epoch 16/25
8000/8000 [==============================] - 1s 117us/step - loss: 0.0029
Epoch 17/25
8000/8000 [==============================] - 1s 124us/step - loss: 0.0029
Epoch 18/25
8000/8000 [==============================] - 1s 124us/step - loss: 0.0029
Epoch 19/25
8000/8000 [==============================] - 1s 124us/step - loss: 0.0029
Epoch 20/25
8000/8000 [==============================] - 1s 126us/step - loss: 0.0029
Epoch 21/25
8000/8000 [==============================] - 1s 128us/step - loss: 0.0029
Epoch 22/25
8000/8000 [==============================] - 1s 123us/step - loss: 0.0028
Epoch 23/25
8000/8000 [==============================] - 1s 126us/step - loss: 0.0028
Epoch 24/25
8000/8000 [==============================] - 1s 126us/step - loss: 0.0028
Epoch 25/25
8000/8000 [==============================] - 1s 123us/step - loss: 0.0028
mse kpca: 0.000432 (0.000460); mse simec: 0.000492 (0.000517)
1500
Epoch 1/25
8000/8000 [==============================] - 1s 179us/step - loss: 0.0174
Epoch 2/25
8000/8000 [==============================] - 1s 138us/step - loss: 0.0086
Epoch 3/25
8000/8000 [==============================] - 1s 140us/step - loss: 0.0046
Epoch 4/25
8000/8000 [==============================] - 1s 138us/step - loss: 0.0036
Epoch 5/25
8000/8000 [==============================] - 1s 141us/step - loss: 0.0032
Epoch 6/25
8000/8000 [==============================] - 1s 141us/step - loss: 0.0030
Epoch 7/25
8000/8000 [==============================] - 1s 142us/step - loss: 0.0030
Epoch 8/25
8000/8000 [==============================] - 1s 142us/step - loss: 0.0029
Epoch 9/25
8000/8000 [==============================] - 1s 142us/step - loss: 0.0029
Epoch 10/25
8000/8000 [==============================] - 1s 146us/step - loss: 0.0029
Epoch 11/25
8000/8000 [==============================] - 1s 137us/step - loss: 0.0029
Epoch 12/25
8000/8000 [==============================] - 1s 143us/step - loss: 0.0029
Epoch 13/25
8000/8000 [==============================] - 1s 145us/step - loss: 0.0029
Epoch 14/25
8000/8000 [==============================] - 1s 143us/step - loss: 0.0029
Epoch 15/25
8000/8000 [==============================] - 1s 142us/step - loss: 0.0029
Epoch 16/25
8000/8000 [==============================] - 1s 143us/step - loss: 0.0028
Epoch 17/25
8000/8000 [==============================] - 1s 139us/step - loss: 0.0028
Epoch 18/25
8000/8000 [==============================] - 1s 140us/step - loss: 0.0028
Epoch 19/25
8000/8000 [==============================] - 1s 144us/step - loss: 0.0028
Epoch 20/25
8000/8000 [==============================] - 1s 144us/step - loss: 0.0028
Epoch 21/25
8000/8000 [==============================] - 1s 141us/step - loss: 0.0028
Epoch 22/25
8000/8000 [==============================] - 1s 139us/step - loss: 0.0028
Epoch 23/25
8000/8000 [==============================] - 1s 136us/step - loss: 0.0028
Epoch 24/25
8000/8000 [==============================] - 1s 136us/step - loss: 0.0028
Epoch 25/25
8000/8000 [==============================] - 1s 140us/step - loss: 0.0028
mse kpca: 0.000432 (0.000460); mse simec: 0.000479 (0.000509)
2500
Epoch 1/25
8000/8000 [==============================] - 2s 269us/step - loss: 0.0171
Epoch 2/25
8000/8000 [==============================] - 2s 208us/step - loss: 0.0089
Epoch 3/25
8000/8000 [==============================] - 2s 205us/step - loss: 0.0047
Epoch 4/25
8000/8000 [==============================] - 2s 202us/step - loss: 0.0035
Epoch 5/25
8000/8000 [==============================] - 2s 205us/step - loss: 0.0030
Epoch 6/25
8000/8000 [==============================] - 2s 206us/step - loss: 0.0029
Epoch 7/25
8000/8000 [==============================] - 2s 208us/step - loss: 0.0029
Epoch 8/25
8000/8000 [==============================] - 2s 205us/step - loss: 0.0029
Epoch 9/25
8000/8000 [==============================] - 2s 208us/step - loss: 0.0029
Epoch 10/25
8000/8000 [==============================] - 2s 205us/step - loss: 0.0028
Epoch 11/25
8000/8000 [==============================] - 2s 203us/step - loss: 0.0028
Epoch 12/25
8000/8000 [==============================] - 2s 203us/step - loss: 0.0028
Epoch 13/25
8000/8000 [==============================] - 2s 205us/step - loss: 0.0028
Epoch 14/25
8000/8000 [==============================] - 2s 203us/step - loss: 0.0028
Epoch 15/25
8000/8000 [==============================] - 2s 203us/step - loss: 0.0028
Epoch 16/25
8000/8000 [==============================] - 2s 208us/step - loss: 0.0028
Epoch 17/25
8000/8000 [==============================] - 2s 206us/step - loss: 0.0028
Epoch 18/25
8000/8000 [==============================] - 2s 208us/step - loss: 0.0028
Epoch 19/25
8000/8000 [==============================] - 2s 204us/step - loss: 0.0028
Epoch 20/25
8000/8000 [==============================] - 2s 204us/step - loss: 0.0027
Epoch 21/25
8000/8000 [==============================] - 2s 206us/step - loss: 0.0027
Epoch 22/25
8000/8000 [==============================] - 2s 204us/step - loss: 0.0027
Epoch 23/25
8000/8000 [==============================] - 2s 203us/step - loss: 0.0027
Epoch 24/25
8000/8000 [==============================] - 2s 206us/step - loss: 0.0027
Epoch 25/25
8000/8000 [==============================] - 2s 206us/step - loss: 0.0027
mse kpca: 0.000432 (0.000460); mse simec: 0.000465 (0.000494)
5000
Epoch 1/25
8000/8000 [==============================] - 5s 646us/step - loss: 0.0173
Epoch 2/25
8000/8000 [==============================] - 4s 461us/step - loss: 0.0101
Epoch 3/25
8000/8000 [==============================] - 4s 461us/step - loss: 0.0048
Epoch 4/25
8000/8000 [==============================] - 4s 459us/step - loss: 0.0033
Epoch 5/25
8000/8000 [==============================] - 4s 460us/step - loss: 0.0030
Epoch 6/25
8000/8000 [==============================] - 4s 463us/step - loss: 0.0029
Epoch 7/25
8000/8000 [==============================] - 4s 457us/step - loss: 0.0029
Epoch 8/25
8000/8000 [==============================] - 4s 457us/step - loss: 0.0028
Epoch 9/25
8000/8000 [==============================] - 4s 456us/step - loss: 0.0028
Epoch 10/25
8000/8000 [==============================] - 4s 462us/step - loss: 0.0028
Epoch 11/25
8000/8000 [==============================] - 4s 465us/step - loss: 0.0028
Epoch 12/25
8000/8000 [==============================] - 4s 465us/step - loss: 0.0028
Epoch 13/25
8000/8000 [==============================] - 4s 462us/step - loss: 0.0028
Epoch 14/25
8000/8000 [==============================] - 4s 465us/step - loss: 0.0028
Epoch 15/25
8000/8000 [==============================] - 4s 465us/step - loss: 0.0028
Epoch 16/25
8000/8000 [==============================] - 4s 464us/step - loss: 0.0028
Epoch 17/25
8000/8000 [==============================] - 4s 464us/step - loss: 0.0028
Epoch 18/25
8000/8000 [==============================] - 4s 463us/step - loss: 0.0028
Epoch 19/25
8000/8000 [==============================] - 4s 465us/step - loss: 0.0027
Epoch 20/25
8000/8000 [==============================] - 4s 464us/step - loss: 0.0027
Epoch 21/25
8000/8000 [==============================] - 4s 467us/step - loss: 0.0027
Epoch 22/25
8000/8000 [==============================] - 4s 465us/step - loss: 0.0027
Epoch 23/25
8000/8000 [==============================] - 4s 463us/step - loss: 0.0027
Epoch 24/25
8000/8000 [==============================] - 4s 464us/step - loss: 0.0027
Epoch 25/25
8000/8000 [==============================] - 4s 466us/step - loss: 0.0027
mse kpca: 0.000432 (0.000460); mse simec: 0.000455 (0.000480)
8000
Epoch 1/25
8000/8000 [==============================] - 11s 1ms/step - loss: 0.0174
Epoch 2/25
8000/8000 [==============================] - 8s 957us/step - loss: 0.0121
Epoch 3/25
8000/8000 [==============================] - 8s 953us/step - loss: 0.0059
Epoch 4/25
8000/8000 [==============================] - 8s 952us/step - loss: 0.0036
Epoch 5/25
8000/8000 [==============================] - 8s 957us/step - loss: 0.0030
Epoch 6/25
8000/8000 [==============================] - 8s 955us/step - loss: 0.0029
Epoch 7/25
8000/8000 [==============================] - 8s 954us/step - loss: 0.0029
Epoch 8/25
8000/8000 [==============================] - 8s 952us/step - loss: 0.0029
Epoch 9/25
8000/8000 [==============================] - 8s 956us/step - loss: 0.0028
Epoch 10/25
8000/8000 [==============================] - 8s 955us/step - loss: 0.0028
Epoch 11/25
8000/8000 [==============================] - 8s 954us/step - loss: 0.0028
Epoch 12/25
8000/8000 [==============================] - 8s 959us/step - loss: 0.0028
Epoch 13/25
8000/8000 [==============================] - 8s 958us/step - loss: 0.0028
Epoch 14/25
8000/8000 [==============================] - 8s 955us/step - loss: 0.0028
Epoch 15/25
8000/8000 [==============================] - 8s 957us/step - loss: 0.0028
Epoch 16/25
8000/8000 [==============================] - 8s 956us/step - loss: 0.0028
Epoch 17/25
8000/8000 [==============================] - 8s 951us/step - loss: 0.0028
Epoch 18/25
8000/8000 [==============================] - 8s 952us/step - loss: 0.0028
Epoch 19/25
8000/8000 [==============================] - 8s 955us/step - loss: 0.0028
Epoch 20/25
8000/8000 [==============================] - 8s 954us/step - loss: 0.0028
Epoch 21/25
8000/8000 [==============================] - 8s 956us/step - loss: 0.0028
Epoch 22/25
8000/8000 [==============================] - 8s 956us/step - loss: 0.0028
Epoch 23/25
8000/8000 [==============================] - 8s 952us/step - loss: 0.0027
Epoch 24/25
8000/8000 [==============================] - 8s 952us/step - loss: 0.0027
Epoch 25/25
8000/8000 [==============================] - 8s 952us/step - loss: 0.0027
mse kpca: 0.000432 (0.000460); mse simec: 0.000450 (0.000478)
targets= [100, 250, 500, 750, 1000, 1500, 2500, 5000, 8000]
mse_k= 0.000431916166063
mse_kt= 0.000459733355453
mse_simec= [0.0006500142759115863, 0.00055137994676708742, 0.00049626462230320772, 0.00048518147857296566, 0.00049232094537341191, 0.00047945665063870091, 0.00046487438714987633, 0.00045453201297292124, 0.00045024873046819688]
mse_simec_test= [0.00069130843017396423, 0.00057953567828591956, 0.00052247096485017364, 0.00051159355885010385, 0.00051716705216015523, 0.00050857207101702862, 0.00049441135229289391, 0.000480097653424192, 0.00047776029172186398]



In [10]:

    
# missing targets
e_dim = 10
n_targets = 1500
np.random.seed(15)
k_mean = np.mean(K_rbf)
mse_ev = []
mse_simec, mse_simec_test = [], []
kpca = KernelPCA(n_components=10, kernel='rbf', gamma=gamma)
X_embed = kpca.fit_transform(X)
X_embed_test = kpca.transform(X_test)
mse_k = check_similarity_match(X_embed, K_rbf)[0]
mse_kt = check_similarity_match(X_embed_test, K_rbf_test)[0]
missing_targets = [0., 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99]
for m in missing_targets:
    print(m)
    K_rbf_noisy = K_rbf.copy()
    K_rbf_noisy[np.random.rand(*K_rbf_noisy.shape)<=m] = -100
    simec = SimilarityEncoder(X.shape[1], e_dim, n_targets, hidden_layers=[(1000, 'tanh')], mask_value=-100,
                              l2_reg=0.00000001, l2_reg_emb=0.00001, l2_reg_out=0.0000001, 
                              s_ll_reg=5., S_ll=K_rbf_noisy[:n_targets,:n_targets], opt=keras.optimizers.Adamax(lr=0.0005))
    simec.fit(X, K_rbf_noisy[:,:n_targets])
    X_embed = simec.transform(X)
    X_embed_test = simec.transform(X_test)
    mse = check_similarity_match(X_embed, K_rbf)[0]
    mse_simec.append(mse)
    mse_t = check_similarity_match(X_embed_test, K_rbf_test)[0]
    mse_simec_test.append(mse_t)
    # eigendecomposition of matrix with missing values (filled in by mean)
    K_rbf_noisy[K_rbf_noisy==-100] = k_mean
    D, V = np.linalg.eig(K_rbf_noisy)
    # embedding based on largest EV
    D1, V1 = D[np.argsort(D)[::-1]], V[:,np.argsort(D)[::-1]]
    X_embed = np.dot(V1.real, np.diag(np.sqrt(np.abs(D1.real))))
    mse_e = check_similarity_match(X_embed[:,:e_dim], K_rbf)[0]
    mse_ev.append(mse_e)
    print("mse kpca: %f (%f); mse evd: %f, mse simec: %f (%f)" % (mse_k, mse_kt, mse_e, mse, mse_t))
keras.backend.clear_session()
print("missing_targets=", missing_targets)
print("mse_k=", mse_k)
print("mse_kt=", mse_kt)
print("mse_ev=", mse_ev)
print("mse_simec=", mse_simec)
print("mse_simec_test=", mse_simec_test)
colors = get_colors(10)
plt.figure();
plt.plot([0, missing_targets[-1]], [mse_k, mse_k], '--', linewidth=0.5, c=colors[8], label='kPCA - no noise');
plt.plot([0, missing_targets[-1]], [mse_kt, mse_kt], '--', linewidth=0.5, c=colors[6], label='kPCA (test) - no noise');
plt.plot(missing_targets, mse_simec, '-o', markersize=3, c=colors[8], label='SimEc');
plt.plot(missing_targets, mse_simec_test, '-o', markersize=3, c=colors[6], label='SimEc (test)');
plt.plot(missing_targets, mse_ev, '-o', markersize=3, c=colors[3], label='Eigendecomposition');
plt.legend(loc=0);
plt.title('MNIST (RBF kernel)');
plt.xticks([0., 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.99], [0., 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.99]);
plt.xlabel('Fraction of Missing Targets')
plt.ylabel('Mean Squared Error of $\hat{S}$')
if savefigs: plt.savefig('fig_rbf_mse_missingt.pdf', dpi=300, bbox_inches="tight")









    



0.0
Epoch 1/25
8000/8000 [==============================] - 2s 199us/step - loss: 0.0176
Epoch 2/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0089
Epoch 3/25
8000/8000 [==============================] - 1s 160us/step - loss: 0.0046
Epoch 4/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0033
Epoch 5/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0030
Epoch 6/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0030
Epoch 7/25
8000/8000 [==============================] - 1s 170us/step - loss: 0.0029
Epoch 8/25
8000/8000 [==============================] - 1s 167us/step - loss: 0.0029
Epoch 9/25
8000/8000 [==============================] - 1s 168us/step - loss: 0.0029
Epoch 10/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0029
Epoch 11/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0029
Epoch 12/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0029
Epoch 13/25
8000/8000 [==============================] - 1s 160us/step - loss: 0.0029
Epoch 14/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0029
Epoch 15/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0029
Epoch 16/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0028
Epoch 17/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0028
Epoch 18/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0028
Epoch 19/25
8000/8000 [==============================] - 1s 159us/step - loss: 0.0028
Epoch 20/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0028
Epoch 21/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0028
Epoch 22/25
8000/8000 [==============================] - 1s 158us/step - loss: 0.0028
Epoch 23/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0028
Epoch 24/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0028
Epoch 25/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0028
mse kpca: 0.000432 (0.000460); mse evd: 0.000432, mse simec: 0.000475 (0.000503)
0.1
Epoch 1/25
8000/8000 [==============================] - 2s 209us/step - loss: 0.0176
Epoch 2/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0088
Epoch 3/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0047
Epoch 4/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0034
Epoch 5/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0030
Epoch 6/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0030
Epoch 7/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0029
Epoch 8/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0029
Epoch 9/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0029
Epoch 10/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0029
Epoch 11/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0029
Epoch 12/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0029
Epoch 13/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0029
Epoch 14/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0029
Epoch 15/25
8000/8000 [==============================] - 1s 160us/step - loss: 0.0028
Epoch 16/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0028
Epoch 17/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0028
Epoch 18/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0028
Epoch 19/25
8000/8000 [==============================] - 1s 158us/step - loss: 0.0028
Epoch 20/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0028
Epoch 21/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0028
Epoch 22/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0028
Epoch 23/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0028
Epoch 24/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0028
Epoch 25/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0028
mse kpca: 0.000432 (0.000460); mse evd: 0.000461, mse simec: 0.000493 (0.000520)
0.2
Epoch 1/25
8000/8000 [==============================] - 2s 213us/step - loss: 0.0178
Epoch 2/25
8000/8000 [==============================] - 1s 160us/step - loss: 0.0092
Epoch 3/25
8000/8000 [==============================] - 1s 160us/step - loss: 0.0048
Epoch 4/25
8000/8000 [==============================] - 1s 158us/step - loss: 0.0035
Epoch 5/25
8000/8000 [==============================] - 1s 160us/step - loss: 0.0031
Epoch 6/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0030
Epoch 7/25
8000/8000 [==============================] - 1s 160us/step - loss: 0.0030
Epoch 8/25
8000/8000 [==============================] - 1s 157us/step - loss: 0.0029
Epoch 9/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0029
Epoch 10/25
8000/8000 [==============================] - 1s 160us/step - loss: 0.0029
Epoch 11/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0029
Epoch 12/25
8000/8000 [==============================] - 1s 160us/step - loss: 0.0029
Epoch 13/25
8000/8000 [==============================] - 1s 160us/step - loss: 0.0029
Epoch 14/25
8000/8000 [==============================] - 1s 159us/step - loss: 0.0029
Epoch 15/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0028
Epoch 16/25
8000/8000 [==============================] - 1s 157us/step - loss: 0.0028
Epoch 17/25
8000/8000 [==============================] - 1s 159us/step - loss: 0.0028
Epoch 18/25
8000/8000 [==============================] - 1s 156us/step - loss: 0.0028
Epoch 19/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0028
Epoch 20/25
8000/8000 [==============================] - 1s 160us/step - loss: 0.0028
Epoch 21/25
8000/8000 [==============================] - 1s 159us/step - loss: 0.0028
Epoch 22/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0028
Epoch 23/25
8000/8000 [==============================] - 1s 157us/step - loss: 0.0028
Epoch 24/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0028
Epoch 25/25
8000/8000 [==============================] - 1s 158us/step - loss: 0.0028
mse kpca: 0.000432 (0.000460); mse evd: 0.000547, mse simec: 0.000474 (0.000500)
0.3
Epoch 1/25
8000/8000 [==============================] - 2s 218us/step - loss: 0.0179
Epoch 2/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0096
Epoch 3/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0051
Epoch 4/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0036
Epoch 5/25
8000/8000 [==============================] - 1s 160us/step - loss: 0.0031
Epoch 6/25
8000/8000 [==============================] - 1s 158us/step - loss: 0.0030
Epoch 7/25
8000/8000 [==============================] - 1s 160us/step - loss: 0.0029
Epoch 8/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0029
Epoch 9/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0029
Epoch 10/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0029
Epoch 11/25
8000/8000 [==============================] - 1s 160us/step - loss: 0.0029
Epoch 12/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0029
Epoch 13/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0029
Epoch 14/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0029
Epoch 15/25
8000/8000 [==============================] - 1s 157us/step - loss: 0.0028
Epoch 16/25
8000/8000 [==============================] - 1s 167us/step - loss: 0.0028
Epoch 17/25
8000/8000 [==============================] - 1s 159us/step - loss: 0.0028
Epoch 18/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0028
Epoch 19/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0028
Epoch 20/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0028
Epoch 21/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0028
Epoch 22/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0028
Epoch 23/25
8000/8000 [==============================] - 1s 160us/step - loss: 0.0028
Epoch 24/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0028
Epoch 25/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0028
mse kpca: 0.000432 (0.000460); mse evd: 0.000688, mse simec: 0.000478 (0.000509)
0.4
Epoch 1/25
8000/8000 [==============================] - 2s 221us/step - loss: 0.0180
Epoch 2/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0100
Epoch 3/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0052
Epoch 4/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0036
Epoch 5/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0031
Epoch 6/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0030
Epoch 7/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0029
Epoch 8/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0029
Epoch 9/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0029
Epoch 10/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0029
Epoch 11/25
8000/8000 [==============================] - 1s 160us/step - loss: 0.0029
Epoch 12/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0029
Epoch 13/25
8000/8000 [==============================] - 1s 159us/step - loss: 0.0029
Epoch 14/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0028
Epoch 15/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0028
Epoch 16/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0028
Epoch 17/25
8000/8000 [==============================] - 1s 158us/step - loss: 0.0028
Epoch 18/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0028
Epoch 19/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0028
Epoch 20/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0028
Epoch 21/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0028
Epoch 22/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0028
Epoch 23/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0028
Epoch 24/25
8000/8000 [==============================] - 1s 160us/step - loss: 0.0028
Epoch 25/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0028
mse kpca: 0.000432 (0.000460); mse evd: 0.000887, mse simec: 0.000475 (0.000505)
0.5
Epoch 1/25
8000/8000 [==============================] - 2s 219us/step - loss: 0.0183
Epoch 2/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0105
Epoch 3/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0054
Epoch 4/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0038
Epoch 5/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0032
Epoch 6/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0030
Epoch 7/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0030
Epoch 8/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0029
Epoch 9/25
8000/8000 [==============================] - 1s 160us/step - loss: 0.0029
Epoch 10/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0029
Epoch 11/25
8000/8000 [==============================] - 1s 160us/step - loss: 0.0029
Epoch 12/25
8000/8000 [==============================] - 1s 159us/step - loss: 0.0029
Epoch 13/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0029
Epoch 14/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0028
Epoch 15/25
8000/8000 [==============================] - 1s 159us/step - loss: 0.0028
Epoch 16/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0028
Epoch 17/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0028
Epoch 18/25
8000/8000 [==============================] - 1s 159us/step - loss: 0.0028
Epoch 19/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0028
Epoch 20/25
8000/8000 [==============================] - 1s 156us/step - loss: 0.0028
Epoch 21/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0028
Epoch 22/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0028
Epoch 23/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0028
Epoch 24/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0028
Epoch 25/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0028
mse kpca: 0.000432 (0.000460); mse evd: 0.001142, mse simec: 0.000476 (0.000501)
0.6
Epoch 1/25
8000/8000 [==============================] - 2s 230us/step - loss: 0.0182
Epoch 2/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0109
Epoch 3/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0060
Epoch 4/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0042
Epoch 5/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0034
Epoch 6/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0032
Epoch 7/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0031
Epoch 8/25
8000/8000 [==============================] - 1s 158us/step - loss: 0.0030
Epoch 9/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0029
Epoch 10/25
8000/8000 [==============================] - 1s 159us/step - loss: 0.0029
Epoch 11/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0029
Epoch 12/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0028
Epoch 13/25
8000/8000 [==============================] - 1s 159us/step - loss: 0.0028
Epoch 14/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0028
Epoch 15/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0028
Epoch 16/25
8000/8000 [==============================] - 1s 155us/step - loss: 0.0028
Epoch 17/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0028
Epoch 18/25
8000/8000 [==============================] - 1s 160us/step - loss: 0.0028
Epoch 19/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0028
Epoch 20/25
8000/8000 [==============================] - 1s 160us/step - loss: 0.0028
Epoch 21/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0028
Epoch 22/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0028
Epoch 23/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0028
Epoch 24/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0027
Epoch 25/25
8000/8000 [==============================] - 1s 158us/step - loss: 0.0027
mse kpca: 0.000432 (0.000460); mse evd: 0.001452, mse simec: 0.000486 (0.000512)
0.7
Epoch 1/25
8000/8000 [==============================] - 2s 229us/step - loss: 0.0188
Epoch 2/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0125
Epoch 3/25
8000/8000 [==============================] - 1s 159us/step - loss: 0.0068
Epoch 4/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0045
Epoch 5/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0035
Epoch 6/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0031
Epoch 7/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0030
Epoch 8/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0029
Epoch 9/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0029
Epoch 10/25
8000/8000 [==============================] - 1s 167us/step - loss: 0.0029
Epoch 11/25
8000/8000 [==============================] - 1s 160us/step - loss: 0.0029
Epoch 12/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0029
Epoch 13/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0028
Epoch 14/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0028
Epoch 15/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0028
Epoch 16/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0028
Epoch 17/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0028
Epoch 18/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0028
Epoch 19/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0028
Epoch 20/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0028
Epoch 21/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0028
Epoch 22/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0028
Epoch 23/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0028
Epoch 24/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0028
Epoch 25/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0028
mse kpca: 0.000432 (0.000460); mse evd: 0.001820, mse simec: 0.000495 (0.000521)
0.8
Epoch 1/25
8000/8000 [==============================] - 2s 233us/step - loss: 0.0190
Epoch 2/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0138
Epoch 3/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0082
Epoch 4/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0052
Epoch 5/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0040
Epoch 6/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0034
Epoch 7/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0031
Epoch 8/25
8000/8000 [==============================] - 1s 169us/step - loss: 0.0030
Epoch 9/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0030
Epoch 10/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0029
Epoch 11/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0029
Epoch 12/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0029
Epoch 13/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0029
Epoch 14/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0028
Epoch 15/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0028
Epoch 16/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0028
Epoch 17/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0028
Epoch 18/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0028
Epoch 19/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0028
Epoch 20/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0028
Epoch 21/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0027
Epoch 22/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0027
Epoch 23/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0027
Epoch 24/25
8000/8000 [==============================] - 1s 169us/step - loss: 0.0027
Epoch 25/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0027
mse kpca: 0.000432 (0.000460); mse evd: 0.002244, mse simec: 0.000492 (0.000524)
0.9
Epoch 1/25
8000/8000 [==============================] - 2s 236us/step - loss: 0.0198
Epoch 2/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0168
Epoch 3/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0123
Epoch 4/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0081
Epoch 5/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0057
Epoch 6/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0043
Epoch 7/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0036
Epoch 8/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0032
Epoch 9/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0030
Epoch 10/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0029
Epoch 11/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0029
Epoch 12/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0028
Epoch 13/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0028
Epoch 14/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0028
Epoch 15/25
8000/8000 [==============================] - 1s 167us/step - loss: 0.0028
Epoch 16/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0028
Epoch 17/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0028
Epoch 18/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0027
Epoch 19/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0027
Epoch 20/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0027
Epoch 21/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0027
Epoch 22/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0027
Epoch 23/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0027
Epoch 24/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0027
Epoch 25/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0027
mse kpca: 0.000432 (0.000460); mse evd: 0.002724, mse simec: 0.000511 (0.000534)
0.95
Epoch 1/25
8000/8000 [==============================] - 2s 233us/step - loss: 0.0201
Epoch 2/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0183
Epoch 3/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0158
Epoch 4/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0124
Epoch 5/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0090
Epoch 6/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0069
Epoch 7/25
8000/8000 [==============================] - 1s 168us/step - loss: 0.0054
Epoch 8/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0044
Epoch 9/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0038
Epoch 10/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0034
Epoch 11/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0032
Epoch 12/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0030
Epoch 13/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0029
Epoch 14/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0028
Epoch 15/25
8000/8000 [==============================] - 1s 167us/step - loss: 0.0027
Epoch 16/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0027
Epoch 17/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0027
Epoch 18/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0027
Epoch 19/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0026
Epoch 20/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0026
Epoch 21/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0026
Epoch 22/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0026
Epoch 23/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0026
Epoch 24/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0026
Epoch 25/25
8000/8000 [==============================] - 1s 167us/step - loss: 0.0026
mse kpca: 0.000432 (0.000460); mse evd: 0.002986, mse simec: 0.000561 (0.000587)
0.99
Epoch 1/25
8000/8000 [==============================] - 2s 235us/step - loss: 0.0204
Epoch 2/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0198
Epoch 3/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0192
Epoch 4/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0184
Epoch 5/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0175
Epoch 6/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0165
Epoch 7/25
8000/8000 [==============================] - 1s 164us/step - loss: 0.0153
Epoch 8/25
8000/8000 [==============================] - 1s 166us/step - loss: 0.0140
Epoch 9/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0124
Epoch 10/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0107
Epoch 11/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0092
Epoch 12/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0080
Epoch 13/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0070
Epoch 14/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0063
Epoch 15/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0057
Epoch 16/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0052
Epoch 17/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0048
Epoch 18/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0045
Epoch 19/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0042
Epoch 20/25
8000/8000 [==============================] - 1s 165us/step - loss: 0.0039
Epoch 21/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0037
Epoch 22/25
8000/8000 [==============================] - 1s 163us/step - loss: 0.0035
Epoch 23/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0033
Epoch 24/25
8000/8000 [==============================] - 1s 161us/step - loss: 0.0032
Epoch 25/25
8000/8000 [==============================] - 1s 162us/step - loss: 0.0031
mse kpca: 0.000432 (0.000460); mse evd: 0.003206, mse simec: 0.001022 (0.001041)
missing_targets= [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99]
mse_k= 0.000431916166063
mse_kt= 0.000459733355453
mse_ev= [0.00043191616606265621, 0.00046100156573390666, 0.00054673007712194874, 0.00068814736772521568, 0.00088744574564896428, 0.0011416700812195764, 0.0014524089034250252, 0.0018201452589175747, 0.0022439750561024179, 0.0027238669909759837, 0.0029858125414759886, 0.0032062093995529027]
mse_simec= [0.00047454208226289349, 0.00049254373412973488, 0.00047370092297916187, 0.00047812885914706613, 0.00047520553052281684, 0.00047615157097048456, 0.00048644320586755247, 0.00049472938152585969, 0.00049210871388930769, 0.00051137378993870264, 0.00056111636076240916, 0.0010224431650533673]
mse_simec_test= [0.00050266797858914661, 0.0005200439153164513, 0.00050018385886777829, 0.00050853190046197712, 0.00050458165532447246, 0.00050118086045028226, 0.0005120218450288263, 0.0005212492787900286, 0.00052364728778227915, 0.00053379759589522143, 0.00058682303443567012, 0.0010414380511512358]

Embedding non-metric similarities and more



In [11]:

    
# load digits
mnist = fetch_mldata('MNIST original', data_home='data')
X_org = mnist.data/255.  # normalize to 0-1
y = np.array(mnist.target, dtype=int)
# only use 0 and 7
X_org = X_org[(y==0)|(y==7),:]
y = y[(y==0)|(y==7)]
X = np.array(X_org>=0.5, dtype=int)  # binarize
# randomly subsample 5000 and split in train/test
np.random.seed(42)
n_samples = 5000
rnd_idx = np.random.permutation(X.shape[0])[:n_samples]
X, X_org, y = X[rnd_idx,:], X_org[rnd_idx,:], y[rnd_idx]
# adapt the input data
ss = StandardScaler(with_std=False)
X_tf = ss.fit_transform(X_org)



In [12]:

    
# compute simpson similarity
# s_ij = sum(a==1 & b==1)/min(sum(a==1),sum(b==1))
sum_one = np.tile(np.sum(X, axis=1),(X.shape[0],1))
S = np.dot(X, X.T)/np.minimum(sum_one, sum_one.T, dtype=float)
# center
S = center_K(S)



In [13]:

    
# check out the eigenvalue spectrum - we've got some significant negative eigenvalues!!
eigenvals = np.linalg.eigvalsh(S)[::-1]
print(eigenvals[:10])
print(eigenvals[-10:])
plt.figure();
plt.plot(list(range(1, S.shape[0]+1)), eigenvals, '-o', markersize=3);
plt.plot([1,S.shape[0]],[0,0], 'k--', linewidth=0.5);
plt.xlim(-25, S.shape[0]+15);
plt.title('Eigenvalue Spectrum of S');
if savefigs: plt.savefig('fig_nonmetric_mnist07_evspec.pdf', dpi=300)









    



[ 531.63744181  235.31839787  215.90045878  154.84834661  121.4820996
  111.22105789   94.46401417   75.34526218   64.11198688   61.37841461]
[  -9.46392977  -10.14040328  -13.21915609  -14.55947722  -15.86383925
  -16.48268389  -22.0225176   -35.22054468  -74.67616799 -131.97347336]



In [14]:

    
# compute embedding based on eigenvalues and -vectors
D, V = np.linalg.eig(S)
# regular kpca embedding: take largest EV
D1, V1 = D[np.argsort(D)[::-1]], V[:,np.argsort(D)[::-1]]
X_embed_largest = np.dot(V1, np.diag(np.sqrt(np.abs(D1))))
# feature discovery: based on absolute value of EV, i.e. also take most negative
D2, V2 = D[np.argsort(np.abs(D))[::-1]], V[:,np.argsort(np.abs(D))[::-1]]
X_embed_abs_imag = np.array(np.dot(V2, np.diag(np.sqrt(np.abs(D2)))), dtype=complex)
# to approximate S, dimensions belonging to negative EV need to be imaginary
X_embed_abs_imag[:, D2 < 0] *= 1j
# inspect similarity matrix: it's a combination of two simmats 
# S1 based on positive EV
S1 = np.dot(X_embed_largest[:, D1>=0], X_embed_largest[:, D1>=0].T)
# and S2 based on negative EV
S2 = np.dot(X_embed_largest[:, D1<0], X_embed_largest[:, D1<0].T)
print("S = S1-S2: %r " % np.allclose(S, S1-S2))









    



S = S1-S2: True



In [15]:

    
# plot largest
plot_mnist2(X_embed_largest[:,:2], y, X_original=X, title='Embedding with largest components')
plt.xlabel('1st component (class)', fontsize=18);
plt.ylabel('2nd component', fontsize=18);
fig = plt.gcf()
fig.set_size_inches(15.5, 8.5)
if savefigs: plt.savefig('fig_nonmetric_mnist07_largest.png', dpi=300)



In [16]:

    
# plot smallest
plot_mnist2(X_embed_largest[:,-2:], y, X_original=X, title='Embedding with most negative components')
plt.xlabel('2nd last component', fontsize=18);
plt.ylabel('last component (stroke weight)', fontsize=18);
fig = plt.gcf()
fig.set_size_inches(15.5, 8.5)
if savefigs: plt.savefig('fig_nonmetric_mnist07_smallest.png', dpi=300)



In [17]:

    
# plot most extreme
plot_mnist2(X_embed_largest[:,[0,-1]], y, X_original=X, title='Embedding with most extreme components')
plt.xlabel('1st component (class)', fontsize=18);
plt.ylabel('last component (stroke weight)', fontsize=18);
fig = plt.gcf()
fig.set_size_inches(15.5, 8.5)



In [18]:

    
mse_ev_pos, mse_ev_abs, mse_simec = [], [], []
e_dims = [2, 5, 10, 25, 50, 75, 100]
for e_dim in e_dims:
    print(e_dim)
    # eigenvalue based embedding - largest (positive) eigenvalues
    mse_ev_pos.append(check_similarity_match(X_embed_largest[:,:e_dim], S)[0])
    # absolute largest eigenvalues
    mse_ev_abs.append(check_similarity_match(X_embed_abs_imag[:,:e_dim], S)[0])
    # simec embedding and prediction
    simec = SimilarityEncoder(X_tf.shape[1], e_dim, S.shape[1], hidden_layers=[(200, 'tanh')],
                              l2_reg=0.0000001, l2_reg_emb=0.0001, l2_reg_out=0.0000001, 
                              opt=keras.optimizers.Adamax(lr=0.0008))
    simec.fit(X_tf, S)
    mse = check_similarity_match(simec.predict(X_tf), S, X_embed_is_S_approx=True)[0]
    mse_simec.append(mse)
    print("mse largest ev: %f; mse abs largest ev: %f; mse simec: %f" % (mse_ev_pos[-1], mse_ev_abs[-1], mse))
keras.backend.clear_session()
print("e_dims=", e_dims)
print("mse_ev_pos=", mse_ev_pos)
print("mse_ev_abs=", mse_ev_abs)
print("mse_simec=", mse_simec)
colors = get_colors(10)
plt.figure();
plt.plot(e_dims, mse_ev_pos, '-o', markersize=3, c=colors[6], label='Largest positive EV');
plt.plot(e_dims, mse_ev_abs, '-o', markersize=3, c=colors[8], label='Absolute largest EV');
plt.plot(e_dims, mse_simec, '-o', markersize=3, c=colors[4], label='SimEc ($YW_l$)');
plt.legend(loc=0);
plt.title('MNIST (non-metric similarity)');
plt.plot([0, e_dims[-1]], [0,0], 'k--', linewidth=0.5);
plt.xticks(e_dims, e_dims);
plt.xlabel('Number of Embedding Dimensions ($d$)')
plt.ylabel('Mean Squared Error of $\hat{S}$')
if savefigs: plt.savefig('fig_nonmetS_mse_edim.pdf', dpi=300, bbox_inches="tight")









    



2
Epoch 1/25
5000/5000 [==============================] - 1s 206us/step - loss: 0.0126
Epoch 2/25
5000/5000 [==============================] - 1s 103us/step - loss: 0.0086
Epoch 3/25
5000/5000 [==============================] - 1s 120us/step - loss: 0.0076
Epoch 4/25
5000/5000 [==============================] - 1s 115us/step - loss: 0.0075
Epoch 5/25
5000/5000 [==============================] - 1s 111us/step - loss: 0.0074
Epoch 6/25
5000/5000 [==============================] - 1s 112us/step - loss: 0.0073
Epoch 7/25
5000/5000 [==============================] - 1s 105us/step - loss: 0.0073
Epoch 8/25
5000/5000 [==============================] - 1s 107us/step - loss: 0.0073
Epoch 9/25
5000/5000 [==============================] - 1s 112us/step - loss: 0.0073
Epoch 10/25
5000/5000 [==============================] - 1s 112us/step - loss: 0.0072
Epoch 11/25
5000/5000 [==============================] - 1s 107us/step - loss: 0.0072
Epoch 12/25
5000/5000 [==============================] - 1s 109us/step - loss: 0.0072
Epoch 13/25
5000/5000 [==============================] - 1s 109us/step - loss: 0.0072
Epoch 14/25
5000/5000 [==============================] - 1s 105us/step - loss: 0.0072
Epoch 15/25
5000/5000 [==============================] - 1s 112us/step - loss: 0.0072
Epoch 16/25
5000/5000 [==============================] - 1s 112us/step - loss: 0.0072
Epoch 17/25
5000/5000 [==============================] - 1s 108us/step - loss: 0.0072
Epoch 18/25
5000/5000 [==============================] - 1s 107us/step - loss: 0.0072
Epoch 19/25
5000/5000 [==============================] - 1s 114us/step - loss: 0.0072
Epoch 20/25
5000/5000 [==============================] - 1s 109us/step - loss: 0.0072
Epoch 21/25
5000/5000 [==============================] - 1s 111us/step - loss: 0.0072
Epoch 22/25
5000/5000 [==============================] - 1s 117us/step - loss: 0.0072
Epoch 23/25
5000/5000 [==============================] - 1s 113us/step - loss: 0.0072
Epoch 24/25
5000/5000 [==============================] - 1s 114us/step - loss: 0.0072
Epoch 25/25
5000/5000 [==============================] - 1s 112us/step - loss: 0.0072
mse largest ev: 0.007028; mse abs largest ev: 0.007028; mse simec: 0.007095
5
Epoch 1/25
5000/5000 [==============================] - 1s 135us/step - loss: 0.0112
Epoch 2/25
5000/5000 [==============================] - 1s 109us/step - loss: 0.0058
Epoch 3/25
5000/5000 [==============================] - 1s 110us/step - loss: 0.0046
Epoch 4/25
5000/5000 [==============================] - 1s 110us/step - loss: 0.0042
Epoch 5/25
5000/5000 [==============================] - 1s 104us/step - loss: 0.0040
Epoch 6/25
5000/5000 [==============================] - 1s 107us/step - loss: 0.0040
Epoch 7/25
5000/5000 [==============================] - 1s 101us/step - loss: 0.0039
Epoch 8/25
5000/5000 [==============================] - 1s 110us/step - loss: 0.0039
Epoch 9/25
5000/5000 [==============================] - 1s 111us/step - loss: 0.0039
Epoch 10/25
5000/5000 [==============================] - 1s 109us/step - loss: 0.0038
Epoch 11/25
5000/5000 [==============================] - 1s 111us/step - loss: 0.0038
Epoch 12/25
5000/5000 [==============================] - 1s 110us/step - loss: 0.0038
Epoch 13/25
5000/5000 [==============================] - 1s 111us/step - loss: 0.0038
Epoch 14/25
5000/5000 [==============================] - 1s 102us/step - loss: 0.0038
Epoch 15/25
5000/5000 [==============================] - 1s 106us/step - loss: 0.0038
Epoch 16/25
5000/5000 [==============================] - 1s 105us/step - loss: 0.0038
Epoch 17/25
5000/5000 [==============================] - 1s 109us/step - loss: 0.0038
Epoch 18/25
5000/5000 [==============================] - 1s 109us/step - loss: 0.0037
Epoch 19/25
5000/5000 [==============================] - 1s 109us/step - loss: 0.0037
Epoch 20/25
5000/5000 [==============================] - 1s 109us/step - loss: 0.0037
Epoch 21/25
5000/5000 [==============================] - 1s 113us/step - loss: 0.0037
Epoch 22/25
5000/5000 [==============================] - 1s 111us/step - loss: 0.0037
Epoch 23/25
5000/5000 [==============================] - 1s 119us/step - loss: 0.0037
Epoch 24/25
5000/5000 [==============================] - 1s 110us/step - loss: 0.0037
Epoch 25/25
5000/5000 [==============================] - 1s 101us/step - loss: 0.0037
mse largest ev: 0.003614; mse abs largest ev: 0.003508; mse simec: 0.003618
10
Epoch 1/25
5000/5000 [==============================] - 1s 134us/step - loss: 0.0102
Epoch 2/25
5000/5000 [==============================] - 1s 104us/step - loss: 0.0051
Epoch 3/25
5000/5000 [==============================] - 1s 111us/step - loss: 0.0036
Epoch 4/25
5000/5000 [==============================] - 1s 103us/step - loss: 0.0029
Epoch 5/25
5000/5000 [==============================] - 1s 119us/step - loss: 0.0026
Epoch 6/25
5000/5000 [==============================] - 1s 114us/step - loss: 0.0024
Epoch 7/25
5000/5000 [==============================] - 1s 118us/step - loss: 0.0023
Epoch 8/25
5000/5000 [==============================] - 1s 108us/step - loss: 0.0022
Epoch 9/25
5000/5000 [==============================] - 1s 106us/step - loss: 0.0021
Epoch 10/25
5000/5000 [==============================] - 1s 109us/step - loss: 0.0021
Epoch 11/25
5000/5000 [==============================] - 1s 106us/step - loss: 0.0021
Epoch 12/25
5000/5000 [==============================] - 1s 118us/step - loss: 0.0021
Epoch 13/25
5000/5000 [==============================] - 1s 110us/step - loss: 0.0020
Epoch 14/25
5000/5000 [==============================] - 1s 108us/step - loss: 0.0020
Epoch 15/25
5000/5000 [==============================] - 1s 111us/step - loss: 0.0020
Epoch 16/25
5000/5000 [==============================] - 1s 109us/step - loss: 0.0020
Epoch 17/25
5000/5000 [==============================] - 1s 104us/step - loss: 0.0020
Epoch 18/25
5000/5000 [==============================] - 1s 104us/step - loss: 0.0020
Epoch 19/25
5000/5000 [==============================] - 1s 114us/step - loss: 0.0020
Epoch 20/25
5000/5000 [==============================] - 1s 107us/step - loss: 0.0020
Epoch 21/25
5000/5000 [==============================] - 1s 112us/step - loss: 0.0020
Epoch 22/25
5000/5000 [==============================] - 1s 104us/step - loss: 0.0020
Epoch 23/25
5000/5000 [==============================] - 1s 109us/step - loss: 0.0020
Epoch 24/25
5000/5000 [==============================] - 1s 111us/step - loss: 0.0019
Epoch 25/25
5000/5000 [==============================] - 1s 115us/step - loss: 0.0019
mse largest ev: 0.002221; mse abs largest ev: 0.001616; mse simec: 0.001825
25
Epoch 1/25
5000/5000 [==============================] - 1s 134us/step - loss: 0.0106
Epoch 2/25
5000/5000 [==============================] - 1s 108us/step - loss: 0.0046
Epoch 3/25
5000/5000 [==============================] - 1s 106us/step - loss: 0.0033
Epoch 4/25
5000/5000 [==============================] - 1s 104us/step - loss: 0.0027
Epoch 5/25
5000/5000 [==============================] - 1s 107us/step - loss: 0.0023
Epoch 6/25
5000/5000 [==============================] - 1s 105us/step - loss: 0.0020
Epoch 7/25
5000/5000 [==============================] - 1s 106us/step - loss: 0.0019
Epoch 8/25
5000/5000 [==============================] - 1s 102us/step - loss: 0.0017
Epoch 9/25
5000/5000 [==============================] - 1s 116us/step - loss: 0.0016
Epoch 10/25
5000/5000 [==============================] - 1s 108us/step - loss: 0.0016
Epoch 11/25
5000/5000 [==============================] - 1s 108us/step - loss: 0.0015
Epoch 12/25
5000/5000 [==============================] - 1s 113us/step - loss: 0.0014
Epoch 13/25
5000/5000 [==============================] - 1s 114us/step - loss: 0.0014
Epoch 14/25
5000/5000 [==============================] - 1s 116us/step - loss: 0.0013
Epoch 15/25
5000/5000 [==============================] - 1s 105us/step - loss: 0.0012
Epoch 16/25
5000/5000 [==============================] - 1s 104us/step - loss: 0.0012
Epoch 17/25
5000/5000 [==============================] - 1s 101us/step - loss: 0.0011
Epoch 18/25
5000/5000 [==============================] - 1s 114us/step - loss: 0.0011
Epoch 19/25
5000/5000 [==============================] - 1s 115us/step - loss: 0.0011
Epoch 20/25
5000/5000 [==============================] - 1s 114us/step - loss: 0.0010
Epoch 21/25
5000/5000 [==============================] - 1s 116us/step - loss: 0.0010
Epoch 22/25
5000/5000 [==============================] - 1s 115us/step - loss: 9.9954e-04
Epoch 23/25
5000/5000 [==============================] - 1s 120us/step - loss: 9.8047e-04
Epoch 24/25
5000/5000 [==============================] - 1s 109us/step - loss: 9.6480e-04
Epoch 25/25
5000/5000 [==============================] - 1s 112us/step - loss: 9.5122e-04
mse largest ev: 0.001406; mse abs largest ev: 0.000503; mse simec: 0.000757
50
Epoch 1/25
5000/5000 [==============================] - 1s 152us/step - loss: 0.0117
Epoch 2/25
5000/5000 [==============================] - 1s 124us/step - loss: 0.0051
Epoch 3/25
5000/5000 [==============================] - 1s 121us/step - loss: 0.0033
Epoch 4/25
5000/5000 [==============================] - 1s 115us/step - loss: 0.0025
Epoch 5/25
5000/5000 [==============================] - 1s 119us/step - loss: 0.0020
Epoch 6/25
5000/5000 [==============================] - 1s 112us/step - loss: 0.0018
Epoch 7/25
5000/5000 [==============================] - 1s 114us/step - loss: 0.0016
Epoch 8/25
5000/5000 [==============================] - 1s 114us/step - loss: 0.0015
Epoch 9/25
5000/5000 [==============================] - 1s 116us/step - loss: 0.0014
Epoch 10/25
5000/5000 [==============================] - 1s 112us/step - loss: 0.0014
Epoch 11/25
5000/5000 [==============================] - 1s 119us/step - loss: 0.0013
Epoch 12/25
5000/5000 [==============================] - 1s 123us/step - loss: 0.0012
Epoch 13/25
5000/5000 [==============================] - 1s 121us/step - loss: 0.0011
Epoch 14/25
5000/5000 [==============================] - 1s 116us/step - loss: 0.0011
Epoch 15/25
5000/5000 [==============================] - 1s 113us/step - loss: 0.0011
Epoch 16/25
5000/5000 [==============================] - 1s 120us/step - loss: 0.0010
Epoch 17/25
5000/5000 [==============================] - 1s 118us/step - loss: 9.8586e-04
Epoch 18/25
5000/5000 [==============================] - 1s 118us/step - loss: 9.6174e-04
Epoch 19/25
5000/5000 [==============================] - 1s 113us/step - loss: 9.3861e-04
Epoch 20/25
5000/5000 [==============================] - 1s 117us/step - loss: 9.1857e-04
Epoch 21/25
5000/5000 [==============================] - 1s 118us/step - loss: 8.9960e-04
Epoch 22/25
5000/5000 [==============================] - 1s 111us/step - loss: 8.8330e-04
Epoch 23/25
5000/5000 [==============================] - 1s 112us/step - loss: 8.6915e-04
Epoch 24/25
5000/5000 [==============================] - 1s 117us/step - loss: 8.5884e-04
Epoch 25/25
5000/5000 [==============================] - 1s 116us/step - loss: 8.4767e-04
mse largest ev: 0.001194; mse abs largest ev: 0.000203; mse simec: 0.000642
75
Epoch 1/25
5000/5000 [==============================] - 1s 157us/step - loss: 0.0132
Epoch 2/25
5000/5000 [==============================] - 1s 113us/step - loss: 0.0056
Epoch 3/25
5000/5000 [==============================] - 1s 117us/step - loss: 0.0034
Epoch 4/25
5000/5000 [==============================] - 1s 112us/step - loss: 0.0024
Epoch 5/25
5000/5000 [==============================] - 1s 112us/step - loss: 0.0020
Epoch 6/25
5000/5000 [==============================] - 1s 119us/step - loss: 0.0017
Epoch 7/25
5000/5000 [==============================] - 1s 121us/step - loss: 0.0016
Epoch 8/25
5000/5000 [==============================] - 1s 122us/step - loss: 0.0015
Epoch 9/25
5000/5000 [==============================] - 1s 117us/step - loss: 0.0014
Epoch 10/25
5000/5000 [==============================] - 1s 112us/step - loss: 0.0013
Epoch 11/25
5000/5000 [==============================] - 1s 115us/step - loss: 0.0012
Epoch 12/25
5000/5000 [==============================] - 1s 120us/step - loss: 0.0011
Epoch 13/25
5000/5000 [==============================] - 1s 115us/step - loss: 0.0011
Epoch 14/25
5000/5000 [==============================] - 1s 124us/step - loss: 0.0010
Epoch 15/25
5000/5000 [==============================] - 1s 119us/step - loss: 0.0010
Epoch 16/25
5000/5000 [==============================] - 1s 113us/step - loss: 9.7296e-04
Epoch 17/25
5000/5000 [==============================] - 1s 113us/step - loss: 9.4915e-04
Epoch 18/25
5000/5000 [==============================] - 1s 114us/step - loss: 9.2723e-04
Epoch 19/25
5000/5000 [==============================] - 1s 114us/step - loss: 9.0525e-04
Epoch 20/25
5000/5000 [==============================] - 1s 115us/step - loss: 8.8935e-04
Epoch 21/25
5000/5000 [==============================] - 1s 118us/step - loss: 8.7798e-04
Epoch 22/25
5000/5000 [==============================] - 1s 123us/step - loss: 8.6498e-04
Epoch 23/25
5000/5000 [==============================] - 1s 113us/step - loss: 8.5311e-04
Epoch 24/25
5000/5000 [==============================] - 1s 121us/step - loss: 8.4207e-04
Epoch 25/25
5000/5000 [==============================] - 1s 117us/step - loss: 8.3589e-04
mse largest ev: 0.001139; mse abs largest ev: 0.000122; mse simec: 0.000625
100
Epoch 1/25
5000/5000 [==============================] - 1s 168us/step - loss: 0.0143
Epoch 2/25
5000/5000 [==============================] - 1s 114us/step - loss: 0.0060
Epoch 3/25
5000/5000 [==============================] - 1s 115us/step - loss: 0.0035
Epoch 4/25
5000/5000 [==============================] - 1s 114us/step - loss: 0.0024
Epoch 5/25
5000/5000 [==============================] - 1s 115us/step - loss: 0.0019
Epoch 6/25
5000/5000 [==============================] - 1s 124us/step - loss: 0.0017
Epoch 7/25
5000/5000 [==============================] - 1s 119us/step - loss: 0.0015
Epoch 8/25
5000/5000 [==============================] - 1s 120us/step - loss: 0.0014
Epoch 9/25
5000/5000 [==============================] - 1s 122us/step - loss: 0.0013
Epoch 10/25
5000/5000 [==============================] - 1s 108us/step - loss: 0.0012
Epoch 11/25
5000/5000 [==============================] - 1s 122us/step - loss: 0.0011
Epoch 12/25
5000/5000 [==============================] - 1s 112us/step - loss: 0.0011
Epoch 13/25
5000/5000 [==============================] - 1s 123us/step - loss: 0.0010
Epoch 14/25
5000/5000 [==============================] - 1s 133us/step - loss: 0.0010
Epoch 15/25
5000/5000 [==============================] - 1s 131us/step - loss: 9.7321e-04
Epoch 16/25
5000/5000 [==============================] - 1s 110us/step - loss: 9.4978e-04
Epoch 17/25
5000/5000 [==============================] - 1s 121us/step - loss: 9.2738e-04
Epoch 18/25
5000/5000 [==============================] - 1s 113us/step - loss: 9.0699e-04
Epoch 19/25
5000/5000 [==============================] - 1s 113us/step - loss: 8.8985e-04
Epoch 20/25
5000/5000 [==============================] - 1s 117us/step - loss: 8.7646e-04
Epoch 21/25
5000/5000 [==============================] - 1s 124us/step - loss: 8.6232e-04
Epoch 22/25
5000/5000 [==============================] - 1s 110us/step - loss: 8.5325e-04
Epoch 23/25
5000/5000 [==============================] - 1s 116us/step - loss: 8.4232e-04
Epoch 24/25
5000/5000 [==============================] - 1s 113us/step - loss: 8.3725e-04
Epoch 25/25
5000/5000 [==============================] - 1s 114us/step - loss: 8.2703e-04
mse largest ev: 0.001114; mse abs largest ev: 0.000084; mse simec: 0.000605
e_dims= [2, 5, 10, 25, 50, 75, 100]
mse_ev_pos= [0.0070284415214137532, 0.003614484758542179, 0.002220559203888737, 0.0014059349474369114, 0.0011937227101523592, 0.0011394693011657769, 0.0011140232152039569]
mse_ev_abs= [0.0070284415214137532, 0.0035081208726048716, 0.001615924360152513, 0.00050340866943100943, 0.00020274532681751199, 0.00012156326992138027, 8.3828831705861297e-05]
mse_simec= [0.0070949120755892216, 0.0036179846559733532, 0.0018251555717895896, 0.0007566822138752715, 0.00064205181640328312, 0.00062472942895032781, 0.00060524388505160112]



In [19]:

    
# normalize individual similarity matrices by largest EV and add/stack
S1 /= np.linalg.norm(S1, ord=2)
S2 /= np.linalg.norm(S2, ord=2)
S1pS2 = S1 + S2
S1sS2 = np.stack([S1, S2], axis=2)
# normalize all to be within a reasonable range
m = np.max(np.abs(S1sS2))
S1 /= m
S2 /= m
S1pS2 /= m
S1sS2 /= m



In [20]:

    
# compute eigenvalue based embeddings for S1 and S2
D, V = np.linalg.eig(S1)
D1, V1 = D[np.argsort(D)[::-1]], V[:,np.argsort(D)[::-1]]
X_embed_s1 = np.dot(V1, np.diag(np.sqrt(np.abs(D1))))
D, V = np.linalg.eig(S2)
D1, V1 = D[np.argsort(D)[::-1]], V[:,np.argsort(D)[::-1]]
X_embed_s2 = np.dot(V1, np.diag(np.sqrt(np.abs(D1))))



In [21]:

    
# by embedding the data with a simec based on the max ev normed added simmats
# we can preserve both features
simec = SimilarityEncoder(X_tf.shape[1], 2, S1pS2.shape[1], hidden_layers=[(200, 'tanh')],
                          l2_reg=0.0000001, l2_reg_emb=0.0001, l2_reg_out=0.0000001,
                          s_ll_reg=1., S_ll=S1pS2,
                          opt=keras.optimizers.Adamax(lr=0.0008))
simec.fit(X_tf, S1pS2)
X_embed = simec.transform(X_tf)
plot_mnist2(X_embed, y, X_original=X, title='Embedding with SimEc')
plt.xlabel('1st component', fontsize=18);
plt.ylabel('2nd component', fontsize=18);
fig = plt.gcf();
fig.set_size_inches(15.5, 8.5);
if savefigs: plt.savefig('fig_nonmetric_mnist07_simec.png', dpi=300)









    



Epoch 1/25
5000/5000 [==============================] - 4s 718us/step - loss: 0.0045
Epoch 2/25
5000/5000 [==============================] - 2s 451us/step - loss: 0.0037
Epoch 3/25
5000/5000 [==============================] - 2s 451us/step - loss: 0.0036
Epoch 4/25
5000/5000 [==============================] - 2s 454us/step - loss: 0.0035
Epoch 5/25
5000/5000 [==============================] - 2s 449us/step - loss: 0.0033
Epoch 6/25
5000/5000 [==============================] - 2s 451us/step - loss: 0.0031
Epoch 7/25
5000/5000 [==============================] - 2s 451us/step - loss: 0.0028
Epoch 8/25
5000/5000 [==============================] - 2s 450us/step - loss: 0.0025
Epoch 9/25
5000/5000 [==============================] - 2s 452us/step - loss: 0.0022
Epoch 10/25
5000/5000 [==============================] - 2s 453us/step - loss: 0.0021
Epoch 11/25
5000/5000 [==============================] - 2s 455us/step - loss: 0.0021
Epoch 12/25
5000/5000 [==============================] - 2s 454us/step - loss: 0.0020
Epoch 13/25
5000/5000 [==============================] - 2s 451us/step - loss: 0.0020
Epoch 14/25
5000/5000 [==============================] - 2s 450us/step - loss: 0.0020
Epoch 15/25
5000/5000 [==============================] - 2s 451us/step - loss: 0.0020
Epoch 16/25
5000/5000 [==============================] - 2s 451us/step - loss: 0.0020
Epoch 17/25
5000/5000 [==============================] - 2s 453us/step - loss: 0.0020
Epoch 18/25
5000/5000 [==============================] - 2s 449us/step - loss: 0.0020
Epoch 19/25
5000/5000 [==============================] - 2s 447us/step - loss: 0.0020
Epoch 20/25
5000/5000 [==============================] - 2s 454us/step - loss: 0.0020
Epoch 21/25
5000/5000 [==============================] - 2s 451us/step - loss: 0.0020
Epoch 22/25
5000/5000 [==============================] - 2s 453us/step - loss: 0.0020
Epoch 23/25
5000/5000 [==============================] - 2s 450us/step - loss: 0.0020
Epoch 24/25
5000/5000 [==============================] - 2s 450us/step - loss: 0.0020
Epoch 25/25
5000/5000 [==============================] - 2s 453us/step - loss: 0.0020



In [22]:

    
# check approximation errors for different numbers of edim
mse_ev_s1, mse_ev_s2 = [], []
mse_simec_pred_s1_s1, mse_simec_pred_s2_s2 = [], []
mse_simec_embed_s1_s1, mse_simec_embed_s2_s2 = [], []
mse_simec_pred_s1ps2_s1, mse_simec_pred_s1ps2_s2 = [], []
mse_simec_embed_s1ps2_s1, mse_simec_embed_s1ps2_s2 = [], []
mse_simec_pred_s1ss2_s1, mse_simec_pred_s1ss2_s2 = [], []
mse_simec_embed_s1ss2_s1, mse_simec_embed_s1ss2_s2 = [], []
e_dims = [2, 5, 10, 25, 50, 75, 100]
for e_dim in e_dims:
    print(e_dim)
    # eigenvalue based embedding for S1 and S2
    mse_ev_s1.append(check_similarity_match(X_embed_s1[:,:e_dim], S1)[0])
    mse_ev_s2.append(check_similarity_match(X_embed_s2[:,:e_dim], S2)[0])
    # simec embedding and prediction for S1
    simec = SimilarityEncoder(X_tf.shape[1], e_dim, S1.shape[1], hidden_layers=[(200, 'tanh')],
                              l2_reg=0.0000001, l2_reg_emb=0.0001, l2_reg_out=0.0000001,
                              s_ll_reg=1., S_ll=S1,
                              opt=keras.optimizers.Adamax(lr=0.0008))
    simec.fit(X_tf, S1)
    mse_simec_embed_s1_s1.append(check_similarity_match(simec.transform(X_tf), S1)[0])
    mse_simec_pred_s1_s1.append(check_similarity_match(simec.predict(X_tf), S1, X_embed_is_S_approx=True)[0])
    # simec embedding and prediction for S2
    simec = SimilarityEncoder(X_tf.shape[1], e_dim, S2.shape[1], hidden_layers=[(200, 'tanh')],
                              l2_reg=0.0000001, l2_reg_emb=0.0001, l2_reg_out=0.0000001,
                              s_ll_reg=1., S_ll=S2,
                              opt=keras.optimizers.Adamax(lr=0.0008))
    simec.fit(X_tf, S2)
    mse_simec_embed_s2_s2.append(check_similarity_match(simec.transform(X_tf), S2)[0])
    mse_simec_pred_s2_s2.append(check_similarity_match(simec.predict(X_tf), S2, X_embed_is_S_approx=True)[0])
    # simec embedding and prediction for S1pS2
    simec = SimilarityEncoder(X_tf.shape[1], e_dim, S1pS2.shape[1], hidden_layers=[(200, 'tanh')],
                              l2_reg=0.0000001, l2_reg_emb=0.0001, l2_reg_out=0.0000001,
                              s_ll_reg=1., S_ll=S1pS2,
                              opt=keras.optimizers.Adamax(lr=0.0008))
    simec.fit(X_tf, S1pS2)
    X_embed = simec.transform(X_tf)
    S_pred = simec.predict(X_tf)
    mse_simec_embed_s1ps2_s1.append(check_similarity_match(X_embed, S1)[0])
    mse_simec_embed_s1ps2_s2.append(check_similarity_match(X_embed, S2)[0])
    mse_simec_pred_s1ps2_s1.append(check_similarity_match(S_pred, S1, X_embed_is_S_approx=True)[0])
    mse_simec_pred_s1ps2_s2.append(check_similarity_match(S_pred, S2, X_embed_is_S_approx=True)[0])
    # simec embedding and prediction for S1sS2
    simec = SimilarityEncoder(X_tf.shape[1], e_dim, (S1sS2.shape[1], S1sS2.shape[2]), hidden_layers=[(200, 'tanh')],
                              l2_reg=0.0000001, l2_reg_emb=0.0001, l2_reg_out=0.0000001,
                              s_ll_reg=1., S_ll=S1sS2,
                              opt=keras.optimizers.Adamax(lr=0.0008))
    simec.fit(X_tf, S1sS2)
    X_embed = simec.transform(X_tf)
    S_pred = simec.predict(X_tf)
    mse_simec_embed_s1ss2_s1.append(check_similarity_match(X_embed, S1)[0])
    mse_simec_embed_s1ss2_s2.append(check_similarity_match(X_embed, S2)[0])
    mse_simec_pred_s1ss2_s1.append(check_similarity_match(S_pred[:,:,0], S1, X_embed_is_S_approx=True)[0])
    mse_simec_pred_s1ss2_s2.append(check_similarity_match(S_pred[:,:,1], S2, X_embed_is_S_approx=True)[0])
    print("mse S1: ev: %f; s1: %f (%f); s1ps2: %f (%f); s1ss2: %f (%f)" % (mse_ev_s1[-1], 
                                                                           mse_simec_embed_s1_s1[-1], mse_simec_pred_s1_s1[-1], 
                                                                           mse_simec_embed_s1ps2_s1[-1], mse_simec_pred_s1ps2_s1[-1], 
                                                                           mse_simec_embed_s1ss2_s1[-1], mse_simec_pred_s1ss2_s1[-1]))
    print("mse S2: ev: %f; s2: %f (%f); s1ps2: %f (%f); s1ss2: %f (%f)" % (mse_ev_s2[-1], 
                                                                           mse_simec_embed_s2_s2[-1], mse_simec_pred_s2_s2[-1], 
                                                                           mse_simec_embed_s1ps2_s2[-1], mse_simec_pred_s1ps2_s2[-1], 
                                                                           mse_simec_embed_s1ss2_s2[-1], mse_simec_pred_s1ss2_s2[-1]))
    keras.backend.clear_session()
print("e_dims=", e_dims)
print("mse_ev_s1=", mse_ev_s1)
print("mse_ev_s2=", mse_ev_s2)
print("mse_simec_pred_s1_s1=", mse_simec_pred_s1_s1)
print("mse_simec_pred_s2_s2=", mse_simec_pred_s2_s2)
print("mse_simec_embed_s1_s1=", mse_simec_embed_s1_s1)
print("mse_simec_embed_s2_s2=", mse_simec_embed_s2_s2)
print("mse_simec_pred_s1ps2_s1=", mse_simec_pred_s1ps2_s1)
print("mse_simec_pred_s1ps2_s2=", mse_simec_pred_s1ps2_s2)
print("mse_simec_embed_s1ps2_s1=", mse_simec_embed_s1ps2_s1)
print("mse_simec_embed_s1ps2_s2=", mse_simec_embed_s1ps2_s2)
print("mse_simec_pred_s1ss2_s1=", mse_simec_pred_s1ss2_s1)
print("mse_simec_pred_s1ss2_s2=", mse_simec_pred_s1ss2_s2)
print("mse_simec_embed_s1ss2_s1=", mse_simec_embed_s1ss2_s1)
print("mse_simec_embed_s1ss2_s2=", mse_simec_embed_s1ss2_s2)
colors = get_colors(15)
plt.figure();
plt.plot(e_dims, mse_ev_s1, '-o', markersize=3, c=colors[0], label='Eigendecomposition');
plt.plot(e_dims, mse_simec_pred_s1_s1, '-o', markersize=3, c=colors[4], label='SimEc S1 ($YW_l$)');
plt.plot(e_dims, mse_simec_embed_s1_s1, '--o', markersize=3, c=colors[4], label='SimEc S1 ($YY^{\\top}$)');
plt.plot(e_dims, mse_simec_pred_s1ss2_s1, '-o', markersize=3, c=colors[8], label='SimEc S1;S2 ($YW_l$)');
plt.plot(e_dims, mse_simec_embed_s1ss2_s1, '--o', markersize=3, c=colors[8], label='SimEc S1;S2 ($YY^{\\top}$)');
plt.plot(e_dims, mse_simec_pred_s1ps2_s1, '-o', markersize=3, c=colors[12], label='SimEc S1+S2 ($YW_l$)');
plt.plot(e_dims, mse_simec_embed_s1ps2_s1, '--o', markersize=3, c=colors[12], label='SimEc S1+S2 ($YY^{\\top}$)');
plt.legend(bbox_to_anchor=(1.02, 1), loc=2, borderaxespad=0.);
plt.title('Approximating S1');
plt.plot([0, e_dims[-1]], [0,0], 'k--', linewidth=0.5);
plt.xticks(e_dims, e_dims);
plt.xlabel('Number of Embedding Dimensions ($d$)')
plt.ylabel('Mean Squared Error of $\hat{S}$')
if savefigs: plt.savefig('fig_nonmetS1_mse_edim.pdf', dpi=300, bbox_inches="tight")
plt.figure();
plt.plot(e_dims, mse_ev_s2, '-o', markersize=3, c=colors[0], label='Eigendecomposition');
plt.plot(e_dims, mse_simec_pred_s2_s2, '-o', markersize=3, c=colors[4], label='SimEc S2 ($YW_l$)');
plt.plot(e_dims, mse_simec_embed_s2_s2, '--o', markersize=3, c=colors[4], label='SimEc S2 ($YY^{\\top}$)');
plt.plot(e_dims, mse_simec_pred_s1ss2_s2, '-o', markersize=3, c=colors[8], label='SimEc S1;S2 ($YW_l$)');
plt.plot(e_dims, mse_simec_embed_s1ss2_s2, '--o', markersize=3, c=colors[8], label='SimEc S1;S2 ($YY^{\\top}$)');
plt.plot(e_dims, mse_simec_pred_s1ps2_s2, '-o', markersize=3, c=colors[12], label='SimEc S1+S2 ($YW_l$)');
plt.plot(e_dims, mse_simec_embed_s1ps2_s2, '--o', markersize=3, c=colors[12], label='SimEc S1+S2 ($YY^{\\top}$)');
l = plt.legend(bbox_to_anchor=(1.02, 1), loc=2, borderaxespad=0.);
plt.title('Approximating S2');
plt.plot([0, e_dims[-1]], [0,0], 'k--', linewidth=0.5);
plt.xticks(e_dims, e_dims);
plt.xlabel('Number of Embedding Dimensions ($d$)')
plt.ylabel('Mean Squared Error of $\hat{S}$')
if savefigs: plt.savefig('fig_nonmetS2_mse_edim.pdf', dpi=300, bbox_inches="tight", bbox_extra_artists=[l])









    



2
Epoch 1/25
5000/5000 [==============================] - 4s 718us/step - loss: 0.0025
Epoch 2/25
5000/5000 [==============================] - 2s 452us/step - loss: 0.0019
Epoch 3/25
5000/5000 [==============================] - 2s 448us/step - loss: 0.0018
Epoch 4/25
5000/5000 [==============================] - 2s 450us/step - loss: 0.0017
Epoch 5/25
5000/5000 [==============================] - 2s 452us/step - loss: 0.0015
Epoch 6/25
5000/5000 [==============================] - 2s 450us/step - loss: 0.0013
Epoch 7/25
5000/5000 [==============================] - 2s 451us/step - loss: 0.0010
Epoch 8/25
5000/5000 [==============================] - 2s 452us/step - loss: 9.3189e-04
Epoch 9/25
5000/5000 [==============================] - 2s 453us/step - loss: 8.9820e-04
Epoch 10/25
5000/5000 [==============================] - 2s 452us/step - loss: 8.8662e-04
Epoch 11/25
5000/5000 [==============================] - 2s 452us/step - loss: 8.8183e-04
Epoch 12/25
5000/5000 [==============================] - 2s 451us/step - loss: 8.7888e-04
Epoch 13/25
5000/5000 [==============================] - 2s 452us/step - loss: 8.7707e-04
Epoch 14/25
5000/5000 [==============================] - 2s 451us/step - loss: 8.7536e-04
Epoch 15/25
5000/5000 [==============================] - 2s 454us/step - loss: 8.7380e-04
Epoch 16/25
5000/5000 [==============================] - 2s 453us/step - loss: 8.7244e-04
Epoch 17/25
5000/5000 [==============================] - 2s 454us/step - loss: 8.7117e-04
Epoch 18/25
5000/5000 [==============================] - 2s 451us/step - loss: 8.7014e-04
Epoch 19/25
5000/5000 [==============================] - 2s 454us/step - loss: 8.6910e-04
Epoch 20/25
5000/5000 [==============================] - 2s 455us/step - loss: 8.6831e-04
Epoch 21/25
5000/5000 [==============================] - 2s 450us/step - loss: 8.6734e-04
Epoch 22/25
5000/5000 [==============================] - 2s 452us/step - loss: 8.6683e-04
Epoch 23/25
5000/5000 [==============================] - 2s 448us/step - loss: 8.6613e-04
Epoch 24/25
5000/5000 [==============================] - 2s 452us/step - loss: 8.6536e-04
Epoch 25/25
5000/5000 [==============================] - 2s 452us/step - loss: 8.6475e-04
Epoch 1/25
5000/5000 [==============================] - 4s 713us/step - loss: 0.0022
Epoch 2/25
5000/5000 [==============================] - 2s 446us/step - loss: 0.0018
Epoch 3/25
5000/5000 [==============================] - 2s 449us/step - loss: 0.0017
Epoch 4/25
5000/5000 [==============================] - 2s 447us/step - loss: 0.0016
Epoch 5/25
5000/5000 [==============================] - 2s 448us/step - loss: 0.0014
Epoch 6/25
5000/5000 [==============================] - 2s 448us/step - loss: 0.0011
Epoch 7/25
5000/5000 [==============================] - 2s 449us/step - loss: 8.8285e-04
Epoch 8/25
5000/5000 [==============================] - 2s 445us/step - loss: 7.2001e-04
Epoch 9/25
5000/5000 [==============================] - 2s 453us/step - loss: 5.8551e-04
Epoch 10/25
5000/5000 [==============================] - 2s 451us/step - loss: 4.9856e-04
Epoch 11/25
5000/5000 [==============================] - 2s 451us/step - loss: 4.5674e-04
Epoch 12/25
5000/5000 [==============================] - 2s 451us/step - loss: 4.3846e-04
Epoch 13/25
5000/5000 [==============================] - 2s 449us/step - loss: 4.2848e-04
Epoch 14/25
5000/5000 [==============================] - 2s 446us/step - loss: 4.2177e-04
Epoch 15/25
5000/5000 [==============================] - 2s 448us/step - loss: 4.1867e-04
Epoch 16/25
5000/5000 [==============================] - 2s 449us/step - loss: 4.1556e-04
Epoch 17/25
5000/5000 [==============================] - 2s 448us/step - loss: 4.1291e-04
Epoch 18/25
5000/5000 [==============================] - 2s 447us/step - loss: 4.1089e-04
Epoch 19/25
5000/5000 [==============================] - 2s 447us/step - loss: 4.0849e-04
Epoch 20/25
5000/5000 [==============================] - 2s 451us/step - loss: 4.0710e-04
Epoch 21/25
5000/5000 [==============================] - 2s 446us/step - loss: 4.0541e-04
Epoch 22/25
5000/5000 [==============================] - 2s 448us/step - loss: 4.0430e-04
Epoch 23/25
5000/5000 [==============================] - 2s 451us/step - loss: 4.0297e-04
Epoch 24/25
5000/5000 [==============================] - 2s 446us/step - loss: 4.0307e-04
Epoch 25/25
5000/5000 [==============================] - 2s 449us/step - loss: 4.0071e-04
Epoch 1/25
5000/5000 [==============================] - 4s 747us/step - loss: 0.0045
Epoch 2/25
5000/5000 [==============================] - 2s 455us/step - loss: 0.0037
Epoch 3/25
5000/5000 [==============================] - 2s 462us/step - loss: 0.0036
Epoch 4/25
5000/5000 [==============================] - 2s 462us/step - loss: 0.0035
Epoch 5/25
5000/5000 [==============================] - 2s 450us/step - loss: 0.0033
Epoch 6/25
5000/5000 [==============================] - 2s 451us/step - loss: 0.0031
Epoch 7/25
5000/5000 [==============================] - 2s 450us/step - loss: 0.0027
Epoch 8/25
5000/5000 [==============================] - 2s 448us/step - loss: 0.0024
Epoch 9/25
5000/5000 [==============================] - 2s 454us/step - loss: 0.0022
Epoch 10/25
5000/5000 [==============================] - 2s 457us/step - loss: 0.0021
Epoch 11/25
5000/5000 [==============================] - 2s 455us/step - loss: 0.0021
Epoch 12/25
5000/5000 [==============================] - 2s 448us/step - loss: 0.0020
Epoch 13/25
5000/5000 [==============================] - 2s 451us/step - loss: 0.0020
Epoch 14/25
5000/5000 [==============================] - 2s 455us/step - loss: 0.0020
Epoch 15/25
5000/5000 [==============================] - 2s 452us/step - loss: 0.0020
Epoch 16/25
5000/5000 [==============================] - 2s 449us/step - loss: 0.0020
Epoch 17/25
5000/5000 [==============================] - 2s 450us/step - loss: 0.0020
Epoch 18/25
5000/5000 [==============================] - 2s 444us/step - loss: 0.0020
Epoch 19/25
5000/5000 [==============================] - 2s 446us/step - loss: 0.0020
Epoch 20/25
5000/5000 [==============================] - 2s 453us/step - loss: 0.0020
Epoch 21/25
5000/5000 [==============================] - 2s 451us/step - loss: 0.0020
Epoch 22/25
5000/5000 [==============================] - 2s 448us/step - loss: 0.0020
Epoch 23/25
5000/5000 [==============================] - 2s 451us/step - loss: 0.0020
Epoch 24/25
5000/5000 [==============================] - 2s 453us/step - loss: 0.0020
Epoch 25/25
5000/5000 [==============================] - 2s 452us/step - loss: 0.0020
Epoch 1/25
5000/5000 [==============================] - 6s 1ms/step - loss: 0.0035
Epoch 2/25
5000/5000 [==============================] - 4s 783us/step - loss: 0.0030
Epoch 3/25
5000/5000 [==============================] - 4s 789us/step - loss: 0.0026
Epoch 4/25
5000/5000 [==============================] - 4s 784us/step - loss: 0.0019
Epoch 5/25
5000/5000 [==============================] - 4s 783us/step - loss: 0.0016
Epoch 6/25
5000/5000 [==============================] - 4s 787us/step - loss: 0.0016
Epoch 7/25
5000/5000 [==============================] - 4s 791us/step - loss: 0.0015
Epoch 8/25
5000/5000 [==============================] - 4s 788us/step - loss: 0.0015
Epoch 9/25
5000/5000 [==============================] - 4s 785us/step - loss: 0.0015
Epoch 10/25
5000/5000 [==============================] - 4s 785us/step - loss: 0.0015
Epoch 11/25
5000/5000 [==============================] - 4s 788us/step - loss: 0.0014
Epoch 12/25
5000/5000 [==============================] - 4s 786us/step - loss: 0.0014
Epoch 13/25
5000/5000 [==============================] - 4s 788us/step - loss: 0.0013
Epoch 14/25
5000/5000 [==============================] - 4s 785us/step - loss: 0.0013
Epoch 15/25
5000/5000 [==============================] - 4s 786us/step - loss: 0.0013
Epoch 16/25
5000/5000 [==============================] - 4s 787us/step - loss: 0.0013
Epoch 17/25
5000/5000 [==============================] - 4s 785us/step - loss: 0.0013
Epoch 18/25
5000/5000 [==============================] - 4s 791us/step - loss: 0.0013
Epoch 19/25
5000/5000 [==============================] - 4s 785us/step - loss: 0.0013
Epoch 20/25
5000/5000 [==============================] - 4s 796us/step - loss: 0.0013
Epoch 21/25
5000/5000 [==============================] - 4s 806us/step - loss: 0.0013
Epoch 22/25
5000/5000 [==============================] - 4s 800us/step - loss: 0.0013
Epoch 23/25
5000/5000 [==============================] - 4s 796us/step - loss: 0.0013
Epoch 24/25
5000/5000 [==============================] - 4s 796us/step - loss: 0.0013
Epoch 25/25
5000/5000 [==============================] - 4s 794us/step - loss: 0.0013
mse S1: ev: 0.000416; s1: 0.000426 (0.000421); s1ps2: 0.001358 (0.001352); s1ss2: 0.001072 (0.000767)
mse S2: ev: 0.000163; s2: 0.000218 (0.000191); s1ps2: 0.001214 (0.001202); s1ss2: 0.000804 (0.000561)
5
Epoch 1/25
5000/5000 [==============================] - 4s 717us/step - loss: 0.0027
Epoch 2/25
5000/5000 [==============================] - 2s 453us/step - loss: 0.0018
Epoch 3/25
5000/5000 [==============================] - 2s 451us/step - loss: 0.0014
Epoch 4/25
5000/5000 [==============================] - 2s 450us/step - loss: 9.3171e-04
Epoch 5/25
5000/5000 [==============================] - 2s 455us/step - loss: 6.8738e-04
Epoch 6/25
5000/5000 [==============================] - 2s 454us/step - loss: 5.6603e-04
Epoch 7/25
5000/5000 [==============================] - 2s 449us/step - loss: 4.8831e-04
Epoch 8/25
5000/5000 [==============================] - 2s 450us/step - loss: 4.5256e-04
Epoch 9/25
5000/5000 [==============================] - 2s 453us/step - loss: 4.3641e-04
Epoch 10/25
5000/5000 [==============================] - 2s 450us/step - loss: 4.2923e-04
Epoch 11/25
5000/5000 [==============================] - 2s 467us/step - loss: 4.2522e-04
Epoch 12/25
5000/5000 [==============================] - 2s 455us/step - loss: 4.2310e-04
Epoch 13/25
5000/5000 [==============================] - 2s 468us/step - loss: 4.2097e-04
Epoch 14/25
5000/5000 [==============================] - 2s 457us/step - loss: 4.1969e-04
Epoch 15/25
5000/5000 [==============================] - 2s 457us/step - loss: 4.1789e-04
Epoch 16/25
5000/5000 [==============================] - 2s 452us/step - loss: 4.1707e-04
Epoch 17/25
5000/5000 [==============================] - 2s 454us/step - loss: 4.1549e-04
Epoch 18/25
5000/5000 [==============================] - 2s 455us/step - loss: 4.1433e-04
Epoch 19/25
5000/5000 [==============================] - 2s 452us/step - loss: 4.1336e-04
Epoch 20/25
5000/5000 [==============================] - 2s 454us/step - loss: 4.1219e-04
Epoch 21/25
5000/5000 [==============================] - 2s 452us/step - loss: 4.1148e-04
Epoch 22/25
5000/5000 [==============================] - 2s 446us/step - loss: 4.1060e-04
Epoch 23/25
5000/5000 [==============================] - 2s 446us/step - loss: 4.0976e-04
Epoch 24/25
5000/5000 [==============================] - 2s 450us/step - loss: 4.0905e-04
Epoch 25/25
5000/5000 [==============================] - 2s 451us/step - loss: 4.0822e-04
Epoch 1/25
5000/5000 [==============================] - 4s 716us/step - loss: 0.0025
Epoch 2/25
5000/5000 [==============================] - 2s 449us/step - loss: 0.0019
Epoch 3/25
5000/5000 [==============================] - 2s 453us/step - loss: 0.0016
Epoch 4/25
5000/5000 [==============================] - 2s 451us/step - loss: 0.0012
Epoch 5/25
5000/5000 [==============================] - 2s 451us/step - loss: 8.2237e-04
Epoch 6/25
5000/5000 [==============================] - 2s 453us/step - loss: 6.1101e-04
Epoch 7/25
5000/5000 [==============================] - 2s 449us/step - loss: 4.9238e-04
Epoch 8/25
5000/5000 [==============================] - 2s 449us/step - loss: 4.2885e-04
Epoch 9/25
5000/5000 [==============================] - 2s 446us/step - loss: 3.8870e-04
Epoch 10/25
5000/5000 [==============================] - 2s 450us/step - loss: 3.6434e-04
Epoch 11/25
5000/5000 [==============================] - 2s 451us/step - loss: 3.5014e-04
Epoch 12/25
5000/5000 [==============================] - 2s 451us/step - loss: 3.3611e-04
Epoch 13/25
5000/5000 [==============================] - 2s 449us/step - loss: 3.2508e-04
Epoch 14/25
5000/5000 [==============================] - 2s 446us/step - loss: 3.1402e-04
Epoch 15/25
5000/5000 [==============================] - 2s 449us/step - loss: 3.0616e-04
Epoch 16/25
5000/5000 [==============================] - 2s 447us/step - loss: 2.9988e-04
Epoch 17/25
5000/5000 [==============================] - 2s 448us/step - loss: 2.9259e-04
Epoch 18/25
5000/5000 [==============================] - 2s 455us/step - loss: 2.8730e-04
Epoch 19/25
5000/5000 [==============================] - 2s 449us/step - loss: 2.8332e-04
Epoch 20/25
5000/5000 [==============================] - 2s 445us/step - loss: 2.7793e-04
Epoch 21/25
5000/5000 [==============================] - 2s 452us/step - loss: 2.7554e-04
Epoch 22/25
5000/5000 [==============================] - 2s 455us/step - loss: 2.7117e-04
Epoch 23/25
5000/5000 [==============================] - 2s 447us/step - loss: 2.6878e-04
Epoch 24/25
5000/5000 [==============================] - 2s 447us/step - loss: 2.6659e-04
Epoch 25/25
5000/5000 [==============================] - 2s 451us/step - loss: 2.6363e-04
Epoch 1/25
5000/5000 [==============================] - 4s 715us/step - loss: 0.0048
Epoch 2/25
5000/5000 [==============================] - 2s 448us/step - loss: 0.0036
Epoch 3/25
5000/5000 [==============================] - 2s 451us/step - loss: 0.0032
Epoch 4/25
5000/5000 [==============================] - 2s 448us/step - loss: 0.0029
Epoch 5/25
5000/5000 [==============================] - 2s 452us/step - loss: 0.0024
Epoch 6/25
5000/5000 [==============================] - 2s 447us/step - loss: 0.0020
Epoch 7/25
5000/5000 [==============================] - 2s 451us/step - loss: 0.0017
Epoch 8/25
5000/5000 [==============================] - 2s 447us/step - loss: 0.0015
Epoch 9/25
5000/5000 [==============================] - 2s 445us/step - loss: 0.0014
Epoch 10/25
5000/5000 [==============================] - 2s 450us/step - loss: 0.0013
Epoch 11/25
5000/5000 [==============================] - 2s 451us/step - loss: 0.0013
Epoch 12/25
5000/5000 [==============================] - 2s 450us/step - loss: 0.0011
Epoch 13/25
5000/5000 [==============================] - 2s 449us/step - loss: 0.0011
Epoch 14/25
5000/5000 [==============================] - 2s 451us/step - loss: 0.0010
Epoch 15/25
5000/5000 [==============================] - 2s 451us/step - loss: 0.0010
Epoch 16/25
5000/5000 [==============================] - 2s 451us/step - loss: 0.0010
Epoch 17/25
5000/5000 [==============================] - 2s 451us/step - loss: 0.0010
Epoch 18/25
5000/5000 [==============================] - 2s 448us/step - loss: 0.0010
Epoch 19/25
5000/5000 [==============================] - 2s 457us/step - loss: 0.0010
Epoch 20/25
5000/5000 [==============================] - 2s 447us/step - loss: 0.0010
Epoch 21/25
5000/5000 [==============================] - 2s 448us/step - loss: 0.0010
Epoch 22/25
5000/5000 [==============================] - 2s 442us/step - loss: 0.0010
Epoch 23/25
5000/5000 [==============================] - 2s 450us/step - loss: 0.0010
Epoch 24/25
5000/5000 [==============================] - 2s 448us/step - loss: 0.0010
Epoch 25/25
5000/5000 [==============================] - 2s 446us/step - loss: 0.0010
Epoch 1/25
5000/5000 [==============================] - 6s 1ms/step - loss: 0.0037
Epoch 2/25
5000/5000 [==============================] - 4s 785us/step - loss: 0.0027
Epoch 3/25
5000/5000 [==============================] - 4s 791us/step - loss: 0.0018
Epoch 4/25
5000/5000 [==============================] - 4s 786us/step - loss: 0.0013
Epoch 5/25
5000/5000 [==============================] - 4s 787us/step - loss: 0.0011
Epoch 6/25
5000/5000 [==============================] - 4s 793us/step - loss: 9.0845e-04
Epoch 7/25
5000/5000 [==============================] - 4s 794us/step - loss: 8.0288e-04
Epoch 8/25
5000/5000 [==============================] - 4s 784us/step - loss: 7.7091e-04
Epoch 9/25
5000/5000 [==============================] - 4s 798us/step - loss: 7.5653e-04
Epoch 10/25
5000/5000 [==============================] - 4s 797us/step - loss: 7.4643e-04
Epoch 11/25
5000/5000 [==============================] - 4s 831us/step - loss: 7.3563e-04
Epoch 12/25
5000/5000 [==============================] - 4s 826us/step - loss: 7.2145e-04
Epoch 13/25
5000/5000 [==============================] - 4s 823us/step - loss: 7.0928e-04
Epoch 14/25
5000/5000 [==============================] - 4s 804us/step - loss: 7.0323e-04
Epoch 15/25
5000/5000 [==============================] - 4s 810us/step - loss: 6.9925e-04
Epoch 16/25
5000/5000 [==============================] - 4s 832us/step - loss: 6.9709e-04
Epoch 17/25
5000/5000 [==============================] - 4s 797us/step - loss: 6.9536e-04
Epoch 18/25
5000/5000 [==============================] - 4s 798us/step - loss: 6.9371e-04
Epoch 19/25
5000/5000 [==============================] - 4s 824us/step - loss: 6.9225e-04
Epoch 20/25
5000/5000 [==============================] - 4s 806us/step - loss: 6.9042e-04
Epoch 21/25
5000/5000 [==============================] - 4s 802us/step - loss: 6.8933e-04
Epoch 22/25
5000/5000 [==============================] - 4s 804us/step - loss: 6.8788e-04
Epoch 23/25
5000/5000 [==============================] - 4s 802us/step - loss: 6.8689e-04
Epoch 24/25
5000/5000 [==============================] - 4s 795us/step - loss: 6.8553e-04
Epoch 25/25
5000/5000 [==============================] - 4s 797us/step - loss: 6.8395e-04
mse S1: ev: 0.000178; s1: 0.000197 (0.000187); s1ps2: 0.001351 (0.001328); s1ss2: 0.001265 (0.000337)
mse S2: ev: 0.000073; s2: 0.000171 (0.000125); s1ps2: 0.001274 (0.001239); s1ss2: 0.001293 (0.000247)
10
Epoch 1/25
5000/5000 [==============================] - 4s 713us/step - loss: 0.0031
Epoch 2/25
5000/5000 [==============================] - 2s 457us/step - loss: 0.0017
Epoch 3/25
5000/5000 [==============================] - 2s 453us/step - loss: 0.0011
Epoch 4/25
5000/5000 [==============================] - 2s 453us/step - loss: 6.6368e-04
Epoch 5/25
5000/5000 [==============================] - 2s 453us/step - loss: 4.8753e-04
Epoch 6/25
5000/5000 [==============================] - 2s 457us/step - loss: 3.8871e-04
Epoch 7/25
5000/5000 [==============================] - 2s 457us/step - loss: 3.3662e-04
Epoch 8/25
5000/5000 [==============================] - 2s 453us/step - loss: 3.0386e-04
Epoch 9/25
5000/5000 [==============================] - 2s 452us/step - loss: 2.8344e-04
Epoch 10/25
5000/5000 [==============================] - 2s 453us/step - loss: 2.6946e-04
Epoch 11/25
5000/5000 [==============================] - 2s 452us/step - loss: 2.5995e-04
Epoch 12/25
5000/5000 [==============================] - 2s 453us/step - loss: 2.5345e-04
Epoch 13/25
5000/5000 [==============================] - 2s 448us/step - loss: 2.5003e-04
Epoch 14/25
5000/5000 [==============================] - 2s 452us/step - loss: 2.4781e-04
Epoch 15/25
5000/5000 [==============================] - 2s 449us/step - loss: 2.4513e-04
Epoch 16/25
5000/5000 [==============================] - 2s 453us/step - loss: 2.4370e-04
Epoch 17/25
5000/5000 [==============================] - 2s 451us/step - loss: 2.4217e-04
Epoch 18/25
5000/5000 [==============================] - 2s 443us/step - loss: 2.4092e-04
Epoch 19/25
5000/5000 [==============================] - 2s 454us/step - loss: 2.3905e-04
Epoch 20/25
5000/5000 [==============================] - 2s 450us/step - loss: 2.3808e-04
Epoch 21/25
5000/5000 [==============================] - 2s 454us/step - loss: 2.3717e-04
Epoch 22/25
5000/5000 [==============================] - 2s 452us/step - loss: 2.3570e-04
Epoch 23/25
5000/5000 [==============================] - 2s 447us/step - loss: 2.3499e-04
Epoch 24/25
5000/5000 [==============================] - 2s 450us/step - loss: 2.3411e-04
Epoch 25/25
5000/5000 [==============================] - 2s 449us/step - loss: 2.3300e-04
Epoch 1/25
5000/5000 [==============================] - 4s 714us/step - loss: 0.0031
Epoch 2/25
5000/5000 [==============================] - 2s 451us/step - loss: 0.0020
Epoch 3/25
5000/5000 [==============================] - 2s 449us/step - loss: 0.0013
Epoch 4/25
5000/5000 [==============================] - 2s 449us/step - loss: 8.9893e-04
Epoch 5/25
5000/5000 [==============================] - 2s 447us/step - loss: 6.7945e-04
Epoch 6/25
5000/5000 [==============================] - 2s 450us/step - loss: 5.5446e-04
Epoch 7/25
5000/5000 [==============================] - 2s 451us/step - loss: 4.8088e-04
Epoch 8/25
5000/5000 [==============================] - 2s 448us/step - loss: 4.2581e-04
Epoch 9/25
5000/5000 [==============================] - 2s 450us/step - loss: 3.9429e-04
Epoch 10/25
5000/5000 [==============================] - 2s 449us/step - loss: 3.6823e-04
Epoch 11/25
5000/5000 [==============================] - 2s 448us/step - loss: 3.4828e-04
Epoch 12/25
5000/5000 [==============================] - 2s 448us/step - loss: 3.2538e-04
Epoch 13/25
5000/5000 [==============================] - 2s 447us/step - loss: 3.0760e-04
Epoch 14/25
5000/5000 [==============================] - 2s 453us/step - loss: 2.9234e-04
Epoch 15/25
5000/5000 [==============================] - 2s 447us/step - loss: 2.7864e-04
Epoch 16/25
5000/5000 [==============================] - 2s 449us/step - loss: 2.6822e-04
Epoch 17/25
5000/5000 [==============================] - 2s 448us/step - loss: 2.5791e-04
Epoch 18/25
5000/5000 [==============================] - 2s 446us/step - loss: 2.5181e-04
Epoch 19/25
5000/5000 [==============================] - 2s 451us/step - loss: 2.4636e-04
Epoch 20/25
5000/5000 [==============================] - 2s 450us/step - loss: 2.3967e-04
Epoch 21/25
5000/5000 [==============================] - 2s 451us/step - loss: 2.3614e-04
Epoch 22/25
5000/5000 [==============================] - 2s 448us/step - loss: 2.3025e-04
Epoch 23/25
5000/5000 [==============================] - 2s 447us/step - loss: 2.2749e-04
Epoch 24/25
5000/5000 [==============================] - 2s 450us/step - loss: 2.2301e-04
Epoch 25/25
5000/5000 [==============================] - 2s 446us/step - loss: 2.1970e-04
Epoch 1/25
5000/5000 [==============================] - 4s 718us/step - loss: 0.0053
Epoch 2/25
5000/5000 [==============================] - 2s 449us/step - loss: 0.0037
Epoch 3/25
5000/5000 [==============================] - 2s 451us/step - loss: 0.0031
Epoch 4/25
5000/5000 [==============================] - 2s 451us/step - loss: 0.0024
Epoch 5/25
5000/5000 [==============================] - 2s 451us/step - loss: 0.0018
Epoch 6/25
5000/5000 [==============================] - 2s 450us/step - loss: 0.0014
Epoch 7/25
5000/5000 [==============================] - 2s 449us/step - loss: 0.0012
Epoch 8/25
5000/5000 [==============================] - 2s 450us/step - loss: 9.3100e-04
Epoch 9/25
5000/5000 [==============================] - 2s 445us/step - loss: 8.0801e-04
Epoch 10/25
5000/5000 [==============================] - 2s 453us/step - loss: 7.4643e-04
Epoch 11/25
5000/5000 [==============================] - 2s 451us/step - loss: 7.0493e-04
Epoch 12/25
5000/5000 [==============================] - 2s 449us/step - loss: 6.7086e-04
Epoch 13/25
5000/5000 [==============================] - 2s 452us/step - loss: 6.4971e-04
Epoch 14/25
5000/5000 [==============================] - 2s 452us/step - loss: 6.3404e-04
Epoch 15/25
5000/5000 [==============================] - 2s 451us/step - loss: 6.2719e-04
Epoch 16/25
5000/5000 [==============================] - 2s 453us/step - loss: 6.2138e-04
Epoch 17/25
5000/5000 [==============================] - 2s 449us/step - loss: 6.1696e-04
Epoch 18/25
5000/5000 [==============================] - 2s 452us/step - loss: 6.1367e-04
Epoch 19/25
5000/5000 [==============================] - 2s 453us/step - loss: 6.1078e-04
Epoch 20/25
5000/5000 [==============================] - 2s 453us/step - loss: 6.0910e-04
Epoch 21/25
5000/5000 [==============================] - 2s 452us/step - loss: 6.0561e-04
Epoch 22/25
5000/5000 [==============================] - 2s 450us/step - loss: 6.0324e-04
Epoch 23/25
5000/5000 [==============================] - 2s 449us/step - loss: 6.0126e-04
Epoch 24/25
5000/5000 [==============================] - 2s 453us/step - loss: 6.0013e-04
Epoch 25/25
5000/5000 [==============================] - 2s 449us/step - loss: 5.9948e-04
Epoch 1/25
5000/5000 [==============================] - 6s 1ms/step - loss: 0.0042
Epoch 2/25
5000/5000 [==============================] - 4s 801us/step - loss: 0.0024
Epoch 3/25
5000/5000 [==============================] - 4s 802us/step - loss: 0.0014
Epoch 4/25
5000/5000 [==============================] - 4s 797us/step - loss: 9.8273e-04
Epoch 5/25
5000/5000 [==============================] - 4s 795us/step - loss: 7.4956e-04
Epoch 6/25
5000/5000 [==============================] - 4s 792us/step - loss: 6.5841e-04
Epoch 7/25
5000/5000 [==============================] - 4s 802us/step - loss: 5.9904e-04
Epoch 8/25
5000/5000 [==============================] - 4s 800us/step - loss: 5.4291e-04
Epoch 9/25
5000/5000 [==============================] - 4s 797us/step - loss: 5.0690e-04
Epoch 10/25
5000/5000 [==============================] - 4s 800us/step - loss: 4.8450e-04
Epoch 11/25
5000/5000 [==============================] - 4s 792us/step - loss: 4.7122e-04
Epoch 12/25
5000/5000 [==============================] - 4s 797us/step - loss: 4.6236e-04
Epoch 13/25
5000/5000 [==============================] - 4s 799us/step - loss: 4.5628e-04
Epoch 14/25
5000/5000 [==============================] - 4s 795us/step - loss: 4.5159e-04
Epoch 15/25
5000/5000 [==============================] - 4s 800us/step - loss: 4.4583e-04
Epoch 16/25
5000/5000 [==============================] - 4s 798us/step - loss: 4.4079e-04
Epoch 17/25
5000/5000 [==============================] - 4s 794us/step - loss: 4.3682e-04
Epoch 18/25
5000/5000 [==============================] - 4s 790us/step - loss: 4.3345e-04
Epoch 19/25
5000/5000 [==============================] - 4s 796us/step - loss: 4.3011e-04
Epoch 20/25
5000/5000 [==============================] - 4s 797us/step - loss: 4.2701e-04
Epoch 21/25
5000/5000 [==============================] - 4s 802us/step - loss: 4.2445e-04
Epoch 22/25
5000/5000 [==============================] - 4s 793us/step - loss: 4.2309e-04
Epoch 23/25
5000/5000 [==============================] - 4s 789us/step - loss: 4.2095e-04
Epoch 24/25
5000/5000 [==============================] - 4s 803us/step - loss: 4.1943e-04
Epoch 25/25
5000/5000 [==============================] - 4s 795us/step - loss: 4.1768e-04
mse S1: ev: 0.000081; s1: 0.000114 (0.000096); s1ps2: 0.001162 (0.001178); s1ss2: 0.001142 (0.000149)
mse S2: ev: 0.000035; s2: 0.000159 (0.000107); s1ps2: 0.001387 (0.001346); s1ss2: 0.001412 (0.000173)
25
Epoch 1/25
5000/5000 [==============================] - 4s 717us/step - loss: 0.0047
Epoch 2/25
5000/5000 [==============================] - 2s 454us/step - loss: 0.0022
Epoch 3/25
5000/5000 [==============================] - 2s 457us/step - loss: 0.0011
Epoch 4/25
5000/5000 [==============================] - 2s 460us/step - loss: 6.4794e-04
Epoch 5/25
5000/5000 [==============================] - 2s 455us/step - loss: 4.4444e-04
Epoch 6/25
5000/5000 [==============================] - 2s 454us/step - loss: 3.4083e-04
Epoch 7/25
5000/5000 [==============================] - 2s 454us/step - loss: 2.8793e-04
Epoch 8/25
5000/5000 [==============================] - 2s 454us/step - loss: 2.5802e-04
Epoch 9/25
5000/5000 [==============================] - 2s 455us/step - loss: 2.3771e-04
Epoch 10/25
5000/5000 [==============================] - 2s 454us/step - loss: 2.2152e-04
Epoch 11/25
5000/5000 [==============================] - 2s 451us/step - loss: 2.0736e-04
Epoch 12/25
5000/5000 [==============================] - 2s 458us/step - loss: 1.9726e-04
Epoch 13/25
5000/5000 [==============================] - 2s 458us/step - loss: 1.8862e-04
Epoch 14/25
5000/5000 [==============================] - 2s 448us/step - loss: 1.8170e-04
Epoch 15/25
5000/5000 [==============================] - 2s 458us/step - loss: 1.7556e-04
Epoch 16/25
5000/5000 [==============================] - 2s 449us/step - loss: 1.7058e-04
Epoch 17/25
5000/5000 [==============================] - 2s 453us/step - loss: 1.6480e-04
Epoch 18/25
5000/5000 [==============================] - 2s 455us/step - loss: 1.6109e-04
Epoch 19/25
5000/5000 [==============================] - 2s 455us/step - loss: 1.5753e-04
Epoch 20/25
5000/5000 [==============================] - 2s 452us/step - loss: 1.5476e-04
Epoch 21/25
5000/5000 [==============================] - 2s 451us/step - loss: 1.5321e-04
Epoch 22/25
5000/5000 [==============================] - 2s 452us/step - loss: 1.5160e-04
Epoch 23/25
5000/5000 [==============================] - 2s 454us/step - loss: 1.4903e-04
Epoch 24/25
5000/5000 [==============================] - 2s 450us/step - loss: 1.4805e-04
Epoch 25/25
5000/5000 [==============================] - 2s 447us/step - loss: 1.4688e-04
Epoch 1/25
5000/5000 [==============================] - 4s 722us/step - loss: 0.0047
Epoch 2/25
5000/5000 [==============================] - 2s 451us/step - loss: 0.0025
Epoch 3/25
5000/5000 [==============================] - 2s 453us/step - loss: 0.0014
Epoch 4/25
5000/5000 [==============================] - 2s 454us/step - loss: 9.0334e-04
Epoch 5/25
5000/5000 [==============================] - 2s 455us/step - loss: 6.7162e-04
Epoch 6/25
5000/5000 [==============================] - 2s 453us/step - loss: 5.6132e-04
Epoch 7/25
5000/5000 [==============================] - 2s 450us/step - loss: 4.8664e-04
Epoch 8/25
5000/5000 [==============================] - 2s 455us/step - loss: 4.3559e-04
Epoch 9/25
5000/5000 [==============================] - 2s 449us/step - loss: 3.9857e-04
Epoch 10/25
5000/5000 [==============================] - 2s 452us/step - loss: 3.6657e-04
Epoch 11/25
5000/5000 [==============================] - 2s 450us/step - loss: 3.3908e-04
Epoch 12/25
5000/5000 [==============================] - 2s 452us/step - loss: 3.1400e-04
Epoch 13/25
5000/5000 [==============================] - 2s 455us/step - loss: 2.9677e-04
Epoch 14/25
5000/5000 [==============================] - 2s 452us/step - loss: 2.8400e-04
Epoch 15/25
5000/5000 [==============================] - 2s 455us/step - loss: 2.7114e-04
Epoch 16/25
5000/5000 [==============================] - 2s 455us/step - loss: 2.6222e-04
Epoch 17/25
5000/5000 [==============================] - 2s 455us/step - loss: 2.5455e-04
Epoch 18/25
5000/5000 [==============================] - 2s 454us/step - loss: 2.4769e-04
Epoch 19/25
5000/5000 [==============================] - 2s 454us/step - loss: 2.3911e-04
Epoch 20/25
5000/5000 [==============================] - 2s 454us/step - loss: 2.3425e-04
Epoch 21/25
5000/5000 [==============================] - 2s 455us/step - loss: 2.2864e-04
Epoch 22/25
5000/5000 [==============================] - 2s 451us/step - loss: 2.2277e-04
Epoch 23/25
5000/5000 [==============================] - 2s 455us/step - loss: 2.1828e-04
Epoch 24/25
5000/5000 [==============================] - 2s 459us/step - loss: 2.1531e-04
Epoch 25/25
5000/5000 [==============================] - 2s 455us/step - loss: 2.1038e-04
Epoch 1/25
5000/5000 [==============================] - 4s 727us/step - loss: 0.0066
Epoch 2/25
5000/5000 [==============================] - 2s 452us/step - loss: 0.0040
Epoch 3/25
5000/5000 [==============================] - 2s 455us/step - loss: 0.0028
Epoch 4/25
5000/5000 [==============================] - 2s 454us/step - loss: 0.0019
Epoch 5/25
5000/5000 [==============================] - 2s 456us/step - loss: 0.0013
Epoch 6/25
5000/5000 [==============================] - 2s 456us/step - loss: 9.8282e-04
Epoch 7/25
5000/5000 [==============================] - 2s 459us/step - loss: 8.1922e-04
Epoch 8/25
5000/5000 [==============================] - 2s 452us/step - loss: 7.1646e-04
Epoch 9/25
5000/5000 [==============================] - 2s 456us/step - loss: 6.5052e-04
Epoch 10/25
5000/5000 [==============================] - 2s 457us/step - loss: 6.0455e-04
Epoch 11/25
5000/5000 [==============================] - 2s 457us/step - loss: 5.7364e-04
Epoch 12/25
5000/5000 [==============================] - 2s 454us/step - loss: 5.4578e-04
Epoch 13/25
5000/5000 [==============================] - 2s 457us/step - loss: 5.1987e-04
Epoch 14/25
5000/5000 [==============================] - 2s 450us/step - loss: 4.9558e-04
Epoch 15/25
5000/5000 [==============================] - 2s 457us/step - loss: 4.7522e-04
Epoch 16/25
5000/5000 [==============================] - 2s 456us/step - loss: 4.5947e-04
Epoch 17/25
5000/5000 [==============================] - 2s 457us/step - loss: 4.4469e-04
Epoch 18/25
5000/5000 [==============================] - 2s 455us/step - loss: 4.3464e-04
Epoch 19/25
5000/5000 [==============================] - 2s 461us/step - loss: 4.2379e-04
Epoch 20/25
5000/5000 [==============================] - 2s 454us/step - loss: 4.1698e-04
Epoch 21/25
5000/5000 [==============================] - 2s 453us/step - loss: 4.1046e-04
Epoch 22/25
5000/5000 [==============================] - 2s 459us/step - loss: 4.0448e-04
Epoch 23/25
5000/5000 [==============================] - 2s 459us/step - loss: 3.9961e-04
Epoch 24/25
5000/5000 [==============================] - 2s 457us/step - loss: 3.9604e-04
Epoch 25/25
5000/5000 [==============================] - 2s 456us/step - loss: 3.9149e-04
Epoch 1/25
5000/5000 [==============================] - 7s 1ms/step - loss: 0.0055
Epoch 2/25
5000/5000 [==============================] - 4s 806us/step - loss: 0.0024
Epoch 3/25
5000/5000 [==============================] - 4s 813us/step - loss: 0.0012
Epoch 4/25
5000/5000 [==============================] - 4s 801us/step - loss: 7.8998e-04
Epoch 5/25
5000/5000 [==============================] - 4s 812us/step - loss: 6.4101e-04
Epoch 6/25
5000/5000 [==============================] - 4s 812us/step - loss: 5.5055e-04
Epoch 7/25
5000/5000 [==============================] - 4s 812us/step - loss: 4.9347e-04
Epoch 8/25
5000/5000 [==============================] - 4s 806us/step - loss: 4.4353e-04
Epoch 9/25
5000/5000 [==============================] - 4s 812us/step - loss: 4.0698e-04
Epoch 10/25
5000/5000 [==============================] - 4s 808us/step - loss: 3.7958e-04
Epoch 11/25
5000/5000 [==============================] - 4s 807us/step - loss: 3.5747e-04
Epoch 12/25
5000/5000 [==============================] - 4s 807us/step - loss: 3.4079e-04
Epoch 13/25
5000/5000 [==============================] - 4s 807us/step - loss: 3.2952e-04
Epoch 14/25
5000/5000 [==============================] - 4s 805us/step - loss: 3.2020e-04
Epoch 15/25
5000/5000 [==============================] - 4s 807us/step - loss: 3.1295e-04
Epoch 16/25
5000/5000 [==============================] - 4s 806us/step - loss: 3.0626e-04
Epoch 17/25
5000/5000 [==============================] - 4s 805us/step - loss: 2.9963e-04
Epoch 18/25
5000/5000 [==============================] - 4s 807us/step - loss: 2.9561e-04
Epoch 19/25
5000/5000 [==============================] - 4s 802us/step - loss: 2.9118e-04
Epoch 20/25
5000/5000 [==============================] - 4s 804us/step - loss: 2.8890e-04
Epoch 21/25
5000/5000 [==============================] - 4s 796us/step - loss: 2.8577e-04
Epoch 22/25
5000/5000 [==============================] - 4s 801us/step - loss: 2.8331e-04
Epoch 23/25
5000/5000 [==============================] - 4s 808us/step - loss: 2.8144e-04
Epoch 24/25
5000/5000 [==============================] - 4s 810us/step - loss: 2.7867e-04
Epoch 25/25
5000/5000 [==============================] - 4s 812us/step - loss: 2.7723e-04
mse S1: ev: 0.000024; s1: 0.000069 (0.000046); s1ps2: 0.001016 (0.001073); s1ss2: 0.001255 (0.000063)
mse S2: ev: 0.000013; s2: 0.000159 (0.000106); s1ps2: 0.001461 (0.001385); s1ss2: 0.001510 (0.000135)
50
Epoch 1/25
5000/5000 [==============================] - 4s 741us/step - loss: 0.0069
Epoch 2/25
5000/5000 [==============================] - 2s 474us/step - loss: 0.0029
Epoch 3/25
5000/5000 [==============================] - 2s 471us/step - loss: 0.0013
Epoch 4/25
5000/5000 [==============================] - 2s 477us/step - loss: 6.9312e-04
Epoch 5/25
5000/5000 [==============================] - 2s 473us/step - loss: 4.5480e-04
Epoch 6/25
5000/5000 [==============================] - 2s 473us/step - loss: 3.3639e-04
Epoch 7/25
5000/5000 [==============================] - 2s 474us/step - loss: 2.7961e-04
Epoch 8/25
5000/5000 [==============================] - 2s 473us/step - loss: 2.4857e-04
Epoch 9/25
5000/5000 [==============================] - 2s 473us/step - loss: 2.3049e-04
Epoch 10/25
5000/5000 [==============================] - 2s 474us/step - loss: 2.1631e-04
Epoch 11/25
5000/5000 [==============================] - 2s 472us/step - loss: 2.0533e-04
Epoch 12/25
5000/5000 [==============================] - 2s 473us/step - loss: 1.9556e-04
Epoch 13/25
5000/5000 [==============================] - 2s 478us/step - loss: 1.8758e-04
Epoch 14/25
5000/5000 [==============================] - 2s 472us/step - loss: 1.8047e-04
Epoch 15/25
5000/5000 [==============================] - 2s 472us/step - loss: 1.7371e-04
Epoch 16/25
5000/5000 [==============================] - 2s 472us/step - loss: 1.6842e-04
Epoch 17/25
5000/5000 [==============================] - 2s 471us/step - loss: 1.6317e-04
Epoch 18/25
5000/5000 [==============================] - 2s 474us/step - loss: 1.5971e-04
Epoch 19/25
5000/5000 [==============================] - 2s 469us/step - loss: 1.5604e-04
Epoch 20/25
5000/5000 [==============================] - 2s 471us/step - loss: 1.5287e-04
Epoch 21/25
5000/5000 [==============================] - 2s 467us/step - loss: 1.4996e-04
Epoch 22/25
5000/5000 [==============================] - 2s 470us/step - loss: 1.4761e-04
Epoch 23/25
5000/5000 [==============================] - 2s 474us/step - loss: 1.4586e-04
Epoch 24/25
5000/5000 [==============================] - 2s 472us/step - loss: 1.4347e-04
Epoch 25/25
5000/5000 [==============================] - 2s 474us/step - loss: 1.4186e-04
Epoch 1/25
5000/5000 [==============================] - 4s 743us/step - loss: 0.0068
Epoch 2/25
5000/5000 [==============================] - 2s 472us/step - loss: 0.0030
Epoch 3/25
5000/5000 [==============================] - 2s 473us/step - loss: 0.0015
Epoch 4/25
5000/5000 [==============================] - 2s 471us/step - loss: 9.0885e-04
Epoch 5/25
5000/5000 [==============================] - 2s 474us/step - loss: 6.6744e-04
Epoch 6/25
5000/5000 [==============================] - 2s 472us/step - loss: 5.5332e-04
Epoch 7/25
5000/5000 [==============================] - 2s 478us/step - loss: 4.8234e-04
Epoch 8/25
5000/5000 [==============================] - 2s 470us/step - loss: 4.2573e-04
Epoch 9/25
5000/5000 [==============================] - 2s 475us/step - loss: 3.9205e-04
Epoch 10/25
5000/5000 [==============================] - 2s 470us/step - loss: 3.6116e-04
Epoch 11/25
5000/5000 [==============================] - 2s 472us/step - loss: 3.3386e-04
Epoch 12/25
5000/5000 [==============================] - 2s 472us/step - loss: 3.1321e-04
Epoch 13/25
5000/5000 [==============================] - 2s 475us/step - loss: 2.9386e-04
Epoch 14/25
5000/5000 [==============================] - 2s 471us/step - loss: 2.8213e-04
Epoch 15/25
5000/5000 [==============================] - 2s 471us/step - loss: 2.7307e-04
Epoch 16/25
5000/5000 [==============================] - 2s 470us/step - loss: 2.6178e-04
Epoch 17/25
5000/5000 [==============================] - 2s 469us/step - loss: 2.5552e-04
Epoch 18/25
5000/5000 [==============================] - 2s 474us/step - loss: 2.4952e-04
Epoch 19/25
5000/5000 [==============================] - 2s 472us/step - loss: 2.4373e-04
Epoch 20/25
5000/5000 [==============================] - 2s 474us/step - loss: 2.3808e-04
Epoch 21/25
5000/5000 [==============================] - 2s 472us/step - loss: 2.3265e-04
Epoch 22/25
5000/5000 [==============================] - 2s 475us/step - loss: 2.2843e-04
Epoch 23/25
5000/5000 [==============================] - 2s 473us/step - loss: 2.2383e-04
Epoch 24/25
5000/5000 [==============================] - 2s 470us/step - loss: 2.1947e-04
Epoch 25/25
5000/5000 [==============================] - 2s 475us/step - loss: 2.1617e-04
Epoch 1/25
5000/5000 [==============================] - 4s 749us/step - loss: 0.0085
Epoch 2/25
5000/5000 [==============================] - 2s 473us/step - loss: 0.0044
Epoch 3/25
5000/5000 [==============================] - 2s 468us/step - loss: 0.0026
Epoch 4/25
5000/5000 [==============================] - 2s 471us/step - loss: 0.0016
Epoch 5/25
5000/5000 [==============================] - 2s 468us/step - loss: 0.0011
Epoch 6/25
5000/5000 [==============================] - 2s 471us/step - loss: 8.8811e-04
Epoch 7/25
5000/5000 [==============================] - 2s 470us/step - loss: 7.5133e-04
Epoch 8/25
5000/5000 [==============================] - 2s 473us/step - loss: 6.6549e-04
Epoch 9/25
5000/5000 [==============================] - 2s 473us/step - loss: 6.0870e-04
Epoch 10/25
5000/5000 [==============================] - 2s 476us/step - loss: 5.7082e-04
Epoch 11/25
5000/5000 [==============================] - 2s 472us/step - loss: 5.3529e-04
Epoch 12/25
5000/5000 [==============================] - 2s 476us/step - loss: 5.0642e-04
Epoch 13/25
5000/5000 [==============================] - 2s 474us/step - loss: 4.7980e-04
Epoch 14/25
5000/5000 [==============================] - 2s 474us/step - loss: 4.5814e-04
Epoch 15/25
5000/5000 [==============================] - 2s 475us/step - loss: 4.3970e-04
Epoch 16/25
5000/5000 [==============================] - 2s 475us/step - loss: 4.2427e-04
Epoch 17/25
5000/5000 [==============================] - 2s 476us/step - loss: 4.1287e-04
Epoch 18/25
5000/5000 [==============================] - 2s 472us/step - loss: 4.0219e-04
Epoch 19/25
5000/5000 [==============================] - 2s 475us/step - loss: 3.9317e-04
Epoch 20/25
5000/5000 [==============================] - 2s 473us/step - loss: 3.8599e-04
Epoch 21/25
5000/5000 [==============================] - 2s 470us/step - loss: 3.7729e-04
Epoch 22/25
5000/5000 [==============================] - 2s 474us/step - loss: 3.7179e-04
Epoch 23/25
5000/5000 [==============================] - 2s 475us/step - loss: 3.6632e-04
Epoch 24/25
5000/5000 [==============================] - 2s 472us/step - loss: 3.5978e-04
Epoch 25/25
5000/5000 [==============================] - 2s 472us/step - loss: 3.5566e-04
Epoch 1/25
5000/5000 [==============================] - 7s 1ms/step - loss: 0.0071
Epoch 2/25
5000/5000 [==============================] - 4s 853us/step - loss: 0.0025
Epoch 3/25
5000/5000 [==============================] - 4s 852us/step - loss: 0.0011
Epoch 4/25
5000/5000 [==============================] - 4s 851us/step - loss: 7.4267e-04
Epoch 5/25
5000/5000 [==============================] - 4s 852us/step - loss: 5.9990e-04
Epoch 6/25
5000/5000 [==============================] - 4s 856us/step - loss: 5.2293e-04
Epoch 7/25
5000/5000 [==============================] - 4s 850us/step - loss: 4.6603e-04
Epoch 8/25
5000/5000 [==============================] - 4s 854us/step - loss: 4.2277e-04
Epoch 9/25
5000/5000 [==============================] - 4s 852us/step - loss: 3.8609e-04
Epoch 10/25
5000/5000 [==============================] - 4s 851us/step - loss: 3.6020e-04
Epoch 11/25
5000/5000 [==============================] - 4s 849us/step - loss: 3.4104e-04
Epoch 12/25
5000/5000 [==============================] - 4s 850us/step - loss: 3.2638e-04
Epoch 13/25
5000/5000 [==============================] - 4s 854us/step - loss: 3.1616e-04
Epoch 14/25
5000/5000 [==============================] - 4s 856us/step - loss: 3.0550e-04
Epoch 15/25
5000/5000 [==============================] - 4s 853us/step - loss: 2.9847e-04
Epoch 16/25
5000/5000 [==============================] - 4s 849us/step - loss: 2.9151e-04
Epoch 17/25
5000/5000 [==============================] - 4s 849us/step - loss: 2.8524e-04
Epoch 18/25
5000/5000 [==============================] - 4s 853us/step - loss: 2.7935e-04
Epoch 19/25
5000/5000 [==============================] - 4s 857us/step - loss: 2.7548e-04
Epoch 20/25
5000/5000 [==============================] - 4s 852us/step - loss: 2.7188e-04
Epoch 21/25
5000/5000 [==============================] - 4s 847us/step - loss: 2.6785e-04
Epoch 22/25
5000/5000 [==============================] - 4s 848us/step - loss: 2.6495e-04
Epoch 23/25
5000/5000 [==============================] - 4s 853us/step - loss: 2.6186e-04
Epoch 24/25
5000/5000 [==============================] - 4s 849us/step - loss: 2.5949e-04
Epoch 25/25
5000/5000 [==============================] - 4s 848us/step - loss: 2.5783e-04
mse S1: ev: 0.000009; s1: 0.000064 (0.000041); s1ps2: 0.001185 (0.001143); s1ss2: 0.001166 (0.000047)
mse S2: ev: 0.000006; s2: 0.000181 (0.000120); s1ps2: 0.001585 (0.001445); s1ss2: 0.001651 (0.000126)
75
Epoch 1/25
5000/5000 [==============================] - 4s 783us/step - loss: 0.0085
Epoch 2/25
5000/5000 [==============================] - 3s 512us/step - loss: 0.0033
Epoch 3/25
5000/5000 [==============================] - 3s 507us/step - loss: 0.0015
Epoch 4/25
5000/5000 [==============================] - 3s 512us/step - loss: 7.6377e-04
Epoch 5/25
5000/5000 [==============================] - 3s 510us/step - loss: 4.9051e-04
Epoch 6/25
5000/5000 [==============================] - 3s 511us/step - loss: 3.5626e-04
Epoch 7/25
5000/5000 [==============================] - 3s 512us/step - loss: 2.9227e-04
Epoch 8/25
5000/5000 [==============================] - 3s 507us/step - loss: 2.5847e-04
Epoch 9/25
5000/5000 [==============================] - 3s 507us/step - loss: 2.3814e-04
Epoch 10/25
5000/5000 [==============================] - 3s 509us/step - loss: 2.2175e-04
Epoch 11/25
5000/5000 [==============================] - 3s 513us/step - loss: 2.1085e-04
Epoch 12/25
5000/5000 [==============================] - 3s 514us/step - loss: 2.0154e-04
Epoch 13/25
5000/5000 [==============================] - 3s 509us/step - loss: 1.9339e-04
Epoch 14/25
5000/5000 [==============================] - 3s 507us/step - loss: 1.8723e-04
Epoch 15/25
5000/5000 [==============================] - 3s 506us/step - loss: 1.7861e-04
Epoch 16/25
5000/5000 [==============================] - 3s 507us/step - loss: 1.7274e-04
Epoch 17/25
5000/5000 [==============================] - 3s 507us/step - loss: 1.6754e-04
Epoch 18/25
5000/5000 [==============================] - 3s 510us/step - loss: 1.6339e-04
Epoch 19/25
5000/5000 [==============================] - 3s 507us/step - loss: 1.5883e-04
Epoch 20/25
5000/5000 [==============================] - 3s 508us/step - loss: 1.5640e-04
Epoch 21/25
5000/5000 [==============================] - 3s 512us/step - loss: 1.5212e-04
Epoch 22/25
5000/5000 [==============================] - 3s 514us/step - loss: 1.5047e-04
Epoch 23/25
5000/5000 [==============================] - 3s 511us/step - loss: 1.4817e-04
Epoch 24/25
5000/5000 [==============================] - 3s 510us/step - loss: 1.4609e-04
Epoch 25/25
5000/5000 [==============================] - 3s 509us/step - loss: 1.4406e-04
Epoch 1/25
5000/5000 [==============================] - 4s 779us/step - loss: 0.0085
Epoch 2/25
5000/5000 [==============================] - 3s 512us/step - loss: 0.0034
Epoch 3/25
5000/5000 [==============================] - 3s 509us/step - loss: 0.0016
Epoch 4/25
5000/5000 [==============================] - 3s 509us/step - loss: 9.5658e-04
Epoch 5/25
5000/5000 [==============================] - 3s 509us/step - loss: 6.8796e-04
Epoch 6/25
5000/5000 [==============================] - 3s 513us/step - loss: 5.7246e-04
Epoch 7/25
5000/5000 [==============================] - 3s 512us/step - loss: 4.9078e-04
Epoch 8/25
5000/5000 [==============================] - 3s 511us/step - loss: 4.3551e-04
Epoch 9/25
5000/5000 [==============================] - 3s 507us/step - loss: 3.9752e-04
Epoch 10/25
5000/5000 [==============================] - 3s 508us/step - loss: 3.6300e-04
Epoch 11/25
5000/5000 [==============================] - 3s 511us/step - loss: 3.3488e-04
Epoch 12/25
5000/5000 [==============================] - 3s 509us/step - loss: 3.1481e-04
Epoch 13/25
5000/5000 [==============================] - 3s 510us/step - loss: 2.9736e-04
Epoch 14/25
5000/5000 [==============================] - 3s 512us/step - loss: 2.8517e-04
Epoch 15/25
5000/5000 [==============================] - 3s 510us/step - loss: 2.7493e-04
Epoch 16/25
5000/5000 [==============================] - 3s 513us/step - loss: 2.6480e-04
Epoch 17/25
5000/5000 [==============================] - 3s 508us/step - loss: 2.5680e-04
Epoch 18/25
5000/5000 [==============================] - 3s 515us/step - loss: 2.4990e-04
Epoch 19/25
5000/5000 [==============================] - 3s 513us/step - loss: 2.4318e-04
Epoch 20/25
5000/5000 [==============================] - 3s 511us/step - loss: 2.3735e-04
Epoch 21/25
5000/5000 [==============================] - 3s 512us/step - loss: 2.3122e-04
Epoch 22/25
5000/5000 [==============================] - 3s 512us/step - loss: 2.2797e-04
Epoch 23/25
5000/5000 [==============================] - 3s 512us/step - loss: 2.2213e-04
Epoch 24/25
5000/5000 [==============================] - 3s 513us/step - loss: 2.1912e-04
Epoch 25/25
5000/5000 [==============================] - 3s 514us/step - loss: 2.1363e-04
Epoch 1/25
5000/5000 [==============================] - 4s 796us/step - loss: 0.0104
Epoch 2/25
5000/5000 [==============================] - 3s 527us/step - loss: 0.0048
Epoch 3/25
5000/5000 [==============================] - 3s 536us/step - loss: 0.0027
Epoch 4/25
5000/5000 [==============================] - 3s 528us/step - loss: 0.0016
Epoch 5/25
5000/5000 [==============================] - 3s 526us/step - loss: 0.0011
Epoch 6/25
5000/5000 [==============================] - 3s 525us/step - loss: 8.9541e-04
Epoch 7/25
5000/5000 [==============================] - 3s 544us/step - loss: 7.5795e-04
Epoch 8/25
5000/5000 [==============================] - 3s 512us/step - loss: 6.7337e-04
Epoch 9/25
5000/5000 [==============================] - 3s 512us/step - loss: 6.1354e-04
Epoch 10/25
5000/5000 [==============================] - 3s 514us/step - loss: 5.7242e-04
Epoch 11/25
5000/5000 [==============================] - 3s 512us/step - loss: 5.3757e-04
Epoch 12/25
5000/5000 [==============================] - 3s 508us/step - loss: 5.0735e-04
Epoch 13/25
5000/5000 [==============================] - 3s 509us/step - loss: 4.7952e-04
Epoch 14/25
5000/5000 [==============================] - 3s 511us/step - loss: 4.6001e-04
Epoch 15/25
5000/5000 [==============================] - 3s 514us/step - loss: 4.4151e-04
Epoch 16/25
5000/5000 [==============================] - 3s 518us/step - loss: 4.2797e-04
Epoch 17/25
5000/5000 [==============================] - 3s 512us/step - loss: 4.1427e-04
Epoch 18/25
5000/5000 [==============================] - 3s 510us/step - loss: 4.0327e-04
Epoch 19/25
5000/5000 [==============================] - 3s 511us/step - loss: 3.9446e-04
Epoch 20/25
5000/5000 [==============================] - 3s 509us/step - loss: 3.8550e-04
Epoch 21/25
5000/5000 [==============================] - 3s 511us/step - loss: 3.7693e-04
Epoch 22/25
5000/5000 [==============================] - 3s 514us/step - loss: 3.7372e-04
Epoch 23/25
5000/5000 [==============================] - 3s 506us/step - loss: 3.6710e-04
Epoch 24/25
5000/5000 [==============================] - 3s 509us/step - loss: 3.5975e-04
Epoch 25/25
5000/5000 [==============================] - 3s 511us/step - loss: 3.5631e-04
Epoch 1/25
5000/5000 [==============================] - 7s 1ms/step - loss: 0.0084
Epoch 2/25
5000/5000 [==============================] - 5s 928us/step - loss: 0.0026
Epoch 3/25
5000/5000 [==============================] - 5s 930us/step - loss: 0.0011
Epoch 4/25
5000/5000 [==============================] - 5s 929us/step - loss: 7.3039e-04
Epoch 5/25
5000/5000 [==============================] - 5s 932us/step - loss: 5.8138e-04
Epoch 6/25
5000/5000 [==============================] - 5s 929us/step - loss: 5.0331e-04
Epoch 7/25
5000/5000 [==============================] - 5s 930us/step - loss: 4.4974e-04
Epoch 8/25
5000/5000 [==============================] - 5s 932us/step - loss: 4.0633e-04
Epoch 9/25
5000/5000 [==============================] - 5s 938us/step - loss: 3.7247e-04
Epoch 10/25
5000/5000 [==============================] - 5s 929us/step - loss: 3.5063e-04
Epoch 11/25
5000/5000 [==============================] - 5s 934us/step - loss: 3.3187e-04
Epoch 12/25
5000/5000 [==============================] - 5s 929us/step - loss: 3.2032e-04
Epoch 13/25
5000/5000 [==============================] - 5s 929us/step - loss: 3.0904e-04
Epoch 14/25
5000/5000 [==============================] - 5s 930us/step - loss: 3.0143e-04
Epoch 15/25
5000/5000 [==============================] - 5s 930us/step - loss: 2.9207e-04
Epoch 16/25
5000/5000 [==============================] - 5s 932us/step - loss: 2.8673e-04
Epoch 17/25
5000/5000 [==============================] - 5s 933us/step - loss: 2.8246e-04
Epoch 18/25
5000/5000 [==============================] - 5s 930us/step - loss: 2.7677e-04
Epoch 19/25
5000/5000 [==============================] - 5s 928us/step - loss: 2.7353e-04
Epoch 20/25
5000/5000 [==============================] - 5s 931us/step - loss: 2.6980e-04
Epoch 21/25
5000/5000 [==============================] - 5s 932us/step - loss: 2.6607e-04
Epoch 22/25
5000/5000 [==============================] - 5s 930us/step - loss: 2.6260e-04
Epoch 23/25
5000/5000 [==============================] - 5s 929us/step - loss: 2.6126e-04
Epoch 24/25
5000/5000 [==============================] - 5s 928us/step - loss: 2.5825e-04
Epoch 25/25
5000/5000 [==============================] - 5s 929us/step - loss: 2.5544e-04
mse S1: ev: 0.000005; s1: 0.000070 (0.000044); s1ps2: 0.001005 (0.001056); s1ss2: 0.001221 (0.000050)
mse S2: ev: 0.000004; s2: 0.000178 (0.000112); s1ps2: 0.001424 (0.001371); s1ss2: 0.001825 (0.000124)
100
Epoch 1/25
5000/5000 [==============================] - 4s 802us/step - loss: 0.0097
Epoch 2/25
5000/5000 [==============================] - 3s 526us/step - loss: 0.0034
Epoch 3/25
5000/5000 [==============================] - 3s 527us/step - loss: 0.0014
Epoch 4/25
5000/5000 [==============================] - 3s 528us/step - loss: 7.0351e-04
Epoch 5/25
5000/5000 [==============================] - 3s 526us/step - loss: 4.4945e-04
Epoch 6/25
5000/5000 [==============================] - 3s 529us/step - loss: 3.2920e-04
Epoch 7/25
5000/5000 [==============================] - 3s 528us/step - loss: 2.7764e-04
Epoch 8/25
5000/5000 [==============================] - 3s 528us/step - loss: 2.4716e-04
Epoch 9/25
5000/5000 [==============================] - 3s 525us/step - loss: 2.2963e-04
Epoch 10/25
5000/5000 [==============================] - 3s 525us/step - loss: 2.1499e-04
Epoch 11/25
5000/5000 [==============================] - 3s 523us/step - loss: 2.0483e-04
Epoch 12/25
5000/5000 [==============================] - 3s 524us/step - loss: 1.9644e-04
Epoch 13/25
5000/5000 [==============================] - 3s 529us/step - loss: 1.9064e-04
Epoch 14/25
5000/5000 [==============================] - 3s 528us/step - loss: 1.8440e-04
Epoch 15/25
5000/5000 [==============================] - 3s 525us/step - loss: 1.7970e-04
Epoch 16/25
5000/5000 [==============================] - 3s 528us/step - loss: 1.7402e-04
Epoch 17/25
5000/5000 [==============================] - 3s 527us/step - loss: 1.7158e-04
Epoch 18/25
5000/5000 [==============================] - 3s 525us/step - loss: 1.6631e-04
Epoch 19/25
5000/5000 [==============================] - 3s 525us/step - loss: 1.6241e-04
Epoch 20/25
5000/5000 [==============================] - 3s 523us/step - loss: 1.5904e-04
Epoch 21/25
5000/5000 [==============================] - 3s 522us/step - loss: 1.5649e-04
Epoch 22/25
5000/5000 [==============================] - 3s 526us/step - loss: 1.5357e-04
Epoch 23/25
5000/5000 [==============================] - 3s 529us/step - loss: 1.5173e-04
Epoch 24/25
5000/5000 [==============================] - 3s 531us/step - loss: 1.5053e-04
Epoch 25/25
5000/5000 [==============================] - 3s 525us/step - loss: 1.4801e-04
Epoch 1/25
5000/5000 [==============================] - 4s 801us/step - loss: 0.0098
Epoch 2/25
5000/5000 [==============================] - 3s 525us/step - loss: 0.0036
Epoch 3/25
5000/5000 [==============================] - 3s 525us/step - loss: 0.0017
Epoch 4/25
5000/5000 [==============================] - 3s 524us/step - loss: 9.2174e-04
Epoch 5/25
5000/5000 [==============================] - 3s 526us/step - loss: 6.6458e-04
Epoch 6/25
5000/5000 [==============================] - 3s 526us/step - loss: 5.5777e-04
Epoch 7/25
5000/5000 [==============================] - 3s 528us/step - loss: 4.8596e-04
Epoch 8/25
5000/5000 [==============================] - 3s 527us/step - loss: 4.3371e-04
Epoch 9/25
5000/5000 [==============================] - 3s 528us/step - loss: 3.9818e-04
Epoch 10/25
5000/5000 [==============================] - 3s 529us/step - loss: 3.6602e-04
Epoch 11/25
5000/5000 [==============================] - 3s 527us/step - loss: 3.4255e-04
Epoch 12/25
5000/5000 [==============================] - 3s 528us/step - loss: 3.1949e-04
Epoch 13/25
5000/5000 [==============================] - 3s 527us/step - loss: 3.0278e-04
Epoch 14/25
5000/5000 [==============================] - 3s 528us/step - loss: 2.9019e-04
Epoch 15/25
5000/5000 [==============================] - 3s 518us/step - loss: 2.7965e-04
Epoch 16/25
5000/5000 [==============================] - 3s 530us/step - loss: 2.7061e-04
Epoch 17/25
5000/5000 [==============================] - 3s 532us/step - loss: 2.6334e-04
Epoch 18/25
5000/5000 [==============================] - 3s 531us/step - loss: 2.5673e-04
Epoch 19/25
5000/5000 [==============================] - 3s 527us/step - loss: 2.4918e-04
Epoch 20/25
5000/5000 [==============================] - 3s 528us/step - loss: 2.4277e-04
Epoch 21/25
5000/5000 [==============================] - 3s 523us/step - loss: 2.3685e-04
Epoch 22/25
5000/5000 [==============================] - 3s 528us/step - loss: 2.3316e-04
Epoch 23/25
5000/5000 [==============================] - 3s 526us/step - loss: 2.2803e-04
Epoch 24/25
5000/5000 [==============================] - 3s 531us/step - loss: 2.2311e-04
Epoch 25/25
5000/5000 [==============================] - 3s 532us/step - loss: 2.2010e-04
Epoch 1/25
5000/5000 [==============================] - 4s 808us/step - loss: 0.0117
Epoch 2/25
5000/5000 [==============================] - 3s 528us/step - loss: 0.0051
Epoch 3/25
5000/5000 [==============================] - 3s 527us/step - loss: 0.0028
Epoch 4/25
5000/5000 [==============================] - 3s 534us/step - loss: 0.0016
Epoch 5/25
5000/5000 [==============================] - 3s 532us/step - loss: 0.0012
Epoch 6/25
5000/5000 [==============================] - 3s 527us/step - loss: 8.9819e-04
Epoch 7/25
5000/5000 [==============================] - 3s 528us/step - loss: 7.5013e-04
Epoch 8/25
5000/5000 [==============================] - 3s 531us/step - loss: 6.5965e-04
Epoch 9/25
5000/5000 [==============================] - 3s 528us/step - loss: 6.0613e-04
Epoch 10/25
5000/5000 [==============================] - 3s 529us/step - loss: 5.6866e-04
Epoch 11/25
5000/5000 [==============================] - 3s 530us/step - loss: 5.3680e-04
Epoch 12/25
5000/5000 [==============================] - 3s 531us/step - loss: 5.0665e-04
Epoch 13/25
5000/5000 [==============================] - 3s 530us/step - loss: 4.8246e-04
Epoch 14/25
5000/5000 [==============================] - 3s 531us/step - loss: 4.6329e-04
Epoch 15/25
5000/5000 [==============================] - 3s 532us/step - loss: 4.4545e-04
Epoch 16/25
5000/5000 [==============================] - 3s 532us/step - loss: 4.3302e-04
Epoch 17/25
5000/5000 [==============================] - 3s 529us/step - loss: 4.2284e-04
Epoch 18/25
5000/5000 [==============================] - 3s 530us/step - loss: 4.0909e-04
Epoch 19/25
5000/5000 [==============================] - 3s 526us/step - loss: 4.0262e-04
Epoch 20/25
5000/5000 [==============================] - 3s 528us/step - loss: 3.9150e-04
Epoch 21/25
5000/5000 [==============================] - 3s 532us/step - loss: 3.8383e-04
Epoch 22/25
5000/5000 [==============================] - 3s 530us/step - loss: 3.7766e-04
Epoch 23/25
5000/5000 [==============================] - 3s 530us/step - loss: 3.7033e-04
Epoch 24/25
5000/5000 [==============================] - 3s 531us/step - loss: 3.6648e-04
Epoch 25/25
5000/5000 [==============================] - 3s 529us/step - loss: 3.6207e-04
Epoch 1/25
5000/5000 [==============================] - 7s 1ms/step - loss: 0.0095
Epoch 2/25
5000/5000 [==============================] - 5s 963us/step - loss: 0.0027
Epoch 3/25
5000/5000 [==============================] - 5s 962us/step - loss: 0.0011
Epoch 4/25
5000/5000 [==============================] - 5s 960us/step - loss: 7.1576e-04
Epoch 5/25
5000/5000 [==============================] - 5s 962us/step - loss: 5.7201e-04
Epoch 6/25
5000/5000 [==============================] - 5s 962us/step - loss: 4.9734e-04
Epoch 7/25
5000/5000 [==============================] - 5s 960us/step - loss: 4.4578e-04
Epoch 8/25
5000/5000 [==============================] - 5s 961us/step - loss: 4.0650e-04
Epoch 9/25
5000/5000 [==============================] - 5s 969us/step - loss: 3.7391e-04
Epoch 10/25
5000/5000 [==============================] - 5s 970us/step - loss: 3.5166e-04
Epoch 11/25
5000/5000 [==============================] - 5s 964us/step - loss: 3.3674e-04
Epoch 12/25
5000/5000 [==============================] - 5s 963us/step - loss: 3.2471e-04
Epoch 13/25
5000/5000 [==============================] - 5s 965us/step - loss: 3.1532e-04
Epoch 14/25
5000/5000 [==============================] - 5s 967us/step - loss: 3.0529e-04
Epoch 15/25
5000/5000 [==============================] - 5s 963us/step - loss: 2.9759e-04
Epoch 16/25
5000/5000 [==============================] - 5s 963us/step - loss: 2.9197e-04
Epoch 17/25
5000/5000 [==============================] - 5s 963us/step - loss: 2.8702e-04
Epoch 18/25
5000/5000 [==============================] - 5s 967us/step - loss: 2.8360e-04
Epoch 19/25
5000/5000 [==============================] - 5s 967us/step - loss: 2.7672e-04
Epoch 20/25
5000/5000 [==============================] - 5s 966us/step - loss: 2.7496e-04
Epoch 21/25
5000/5000 [==============================] - 5s 966us/step - loss: 2.7091e-04
Epoch 22/25
5000/5000 [==============================] - 5s 965us/step - loss: 2.6877e-04
Epoch 23/25
5000/5000 [==============================] - 5s 962us/step - loss: 2.6559e-04
Epoch 24/25
5000/5000 [==============================] - 5s 966us/step - loss: 2.6445e-04
Epoch 25/25
5000/5000 [==============================] - 5s 963us/step - loss: 2.6040e-04
mse S1: ev: 0.000004; s1: 0.000075 (0.000046); s1ps2: 0.001055 (0.001075); s1ss2: 0.001314 (0.000048)
mse S2: ev: 0.000003; s2: 0.000262 (0.000121); s1ps2: 0.001558 (0.001438); s1ss2: 0.001850 (0.000134)
e_dims= [2, 5, 10, 25, 50, 75, 100]
mse_ev_s1= [0.00041580766572419075, 0.00017781444326033693, 8.064132196174042e-05, 2.3852362593925464e-05, 9.0586561396199184e-06, 5.2765509370027306e-06, 3.5026573531683373e-06]
mse_ev_s2= [0.00016294438577751268, 7.2571827087261005e-05, 3.497854733580489e-05, 1.3450426810926688e-05, 6.1842613723045256e-06, 3.9067165545074946e-06, 2.7746457044144675e-06]
mse_simec_pred_s1_s1= [0.0004210098580904151, 0.00018721682952340762, 9.6153015505545957e-05, 4.6131361037084238e-05, 4.0786571286511945e-05, 4.367505602771167e-05, 4.5546362927005723e-05]
mse_simec_pred_s2_s2= [0.00019068430976037536, 0.00012456496033480332, 0.00010674498060128491, 0.00010604993219498334, 0.00011964764129153218, 0.00011192636741385388, 0.00012112958392145639]
mse_simec_embed_s1_s1= [0.00042589471608628788, 0.00019718074876624281, 0.00011441907598300842, 6.9083534292105556e-05, 6.4292756689985736e-05, 7.0090641989854522e-05, 7.4618879672416177e-05]
mse_simec_embed_s2_s2= [0.00021756651660129181, 0.00017074787974267206, 0.00015915436089450338, 0.0001585020382870159, 0.0001806778319044993, 0.00017764934873318173, 0.00026228955299335274]
mse_simec_pred_s1ps2_s1= [0.0013518028897756332, 0.0013278202018596008, 0.0011778745415686857, 0.001073088720021351, 0.0011432789602966377, 0.0010560220107782652, 0.0010747692213269767]
mse_simec_pred_s1ps2_s2= [0.0012022579330138001, 0.0012385753251988813, 0.0013456195476657807, 0.001384950573293094, 0.0014453397324141394, 0.0013705557601175089, 0.0014381402577968308]
mse_simec_embed_s1ps2_s1= [0.0013576576892623672, 0.0013510659529517393, 0.001161779313230628, 0.0010161708672521822, 0.0011852909434767007, 0.00100521026796025, 0.0010549557162041997]
mse_simec_embed_s1ps2_s2= [0.0012143368110523166, 0.0012737964080922627, 0.0013872599356079571, 0.0014612579684753063, 0.0015845708742662776, 0.0014235193963222179, 0.0015580437729580826]
mse_simec_pred_s1ss2_s1= [0.00076733123842122699, 0.00033657467921257065, 0.00014934257589038218, 6.3012654629248947e-05, 4.6663339780937365e-05, 5.0177520934578857e-05, 4.8457642343067802e-05]
mse_simec_pred_s1ss2_s2= [0.00056107080972676588, 0.00024655250015859377, 0.00017263487991948055, 0.00013539375906702626, 0.00012597351059335301, 0.00012356565826716282, 0.00013392463848558789]
mse_simec_embed_s1ss2_s1= [0.0010723609358901868, 0.0012651342554963253, 0.0011416594435084196, 0.0012554831439762942, 0.0011655849943027, 0.0012206947341795269, 0.0013139327972492519]
mse_simec_embed_s1ss2_s2= [0.00080370746076519875, 0.0012934808900394183, 0.0014122678871294257, 0.0015102986199177076, 0.001650520015159084, 0.0018250662619850124, 0.0018501986454166164]



In [ ]: