In [1]:
import os
import numpy as np
import scipy.stats

from sklearn.metrics import roc_auc_score, average_precision_score

import pmf

In [2]:
train_tracks = list()
with open('train_tracks.txt', 'rb') as f:
    for line in f:
        train_tracks.append(line.strip())
        
test_tracks = list()
with open('test_tracks.txt', 'rb') as f:
    for line in f:
        test_tracks.append(line.strip())
        
tags = list()
with open('voc.txt', 'rb') as f:
    for line in f:
        tags.append(line.strip())

In [3]:
def construct_pred_mask(tags_predicted, predictat):
    n_samples, n_tags = tags_predicted.shape
    rankings = np.argsort(-tags_predicted, axis=1)[:, :predictat]
    tags_predicted_binary = np.zeros_like(tags_predicted, dtype=bool)
    for i in xrange(n_samples):
        tags_predicted_binary[i, rankings[i]] = 1
    return tags_predicted_binary

def per_tag_prec_recall(tags_predicted_binary, tags_true_binary):
    mask = np.logical_and(tags_predicted_binary, tags_true_binary)
    prec = mask.sum(axis=0) / (tags_predicted_binary.sum(axis=0) + np.spacing(1))
    tags_true_count = tags_true_binary.sum(axis=0).astype(float)
    idx = (tags_true_count > 0)
    recall = mask.sum(axis=0)[idx] / tags_true_count[idx]
    return prec, recall


def aroc_ap(tags_true_binary, tags_predicted):
    n_tags = tags_true_binary.shape[1]
    
    auc = list()
    aprec = list()
    for i in xrange(n_tags):
        if np.sum(tags_true_binary[:, i]) != 0:
            auc.append(roc_auc_score(tags_true_binary[:, i], tags_predicted[:, i]))
            aprec.append(average_precision_score(tags_true_binary[:, i], tags_predicted[:, i]))
    return auc, aprec


def print_out_metrics(tags_true_binary, tags_predicted, predictat):
    tags_predicted_binary = construct_pred_mask(tags_predicted, predictat)
    prec, recall = per_tag_prec_recall(tags_predicted_binary, tags_true_binary)
    mprec, mrecall = np.mean(prec), np.mean(recall)
    
    print 'Precision = %.3f (%.3f)' % (mprec, np.std(prec) / sqrt(prec.size))
    print 'Recall = %.3f (%.3f)' % (mrecall, np.std(recall) / sqrt(recall.size))
    print 'F-score = %.3f' % (2 * mprec * mrecall / (mprec + mrecall))

    auc, aprec = aroc_ap(tags_true_binary, tags_predicted)
    print 'AROC = %.3f (%.3f)' % (np.mean(auc), np.std(auc) / sqrt(len(auc)))
    print 'AP = %.3f (%.3f)' % (np.mean(aprec), np.std(aprec) / sqrt(len(aprec)))

In [4]:
# take tracks with at least 20 tags 
y_test = None

test_tracks_selected = list()

for tid in test_tracks:
    tdir = os.path.join('vq_hist', '/'.join(tid[2:5]))
    bot = np.load(os.path.join(tdir, '%s_BoT.npy' % tid))
    if (bot > 0).sum() >= 20:
        test_tracks_selected.append(tid)
        if y_test is None:
            y_test = bot
        else:
            y_test = np.vstack((y_test, bot))

In [5]:
hist(np.sum( (y_test > 0), axis=1), bins=50)
pass


Batch inference on 10K subset


In [6]:
K = 512

In [7]:
n_subset = 10000

np.random.seed(98765)
train_tracks_subset = np.random.choice(train_tracks, size=n_subset, replace=False)

In [8]:
D = K + len(tags)

X = np.empty((n_subset, D), dtype=np.int16)

for (i, tid) in enumerate(train_tracks_subset):
    tdir = os.path.join('vq_hist', '/'.join(tid[2:5]))
    vq = np.load(os.path.join(tdir, '%s_K%d.npy' % (tid, K))).ravel()
    bot = np.load(os.path.join(tdir, '%s_BoT.npy' % tid))
    bot[bot > 0] = 1
    X[i] = np.hstack((vq, bot))

In [9]:
bar(np.arange(D), X[1000])


Out[9]:
<Container object of 1073 artists>

In [10]:
X_test = np.empty((len(test_tracks_selected), K), dtype=int16)

for (i, tid) in enumerate(test_tracks_selected):
    tdir = os.path.join('vq_hist', '/'.join(tid[2:5]))
    vq = np.load(os.path.join(tdir, '%s_K%d.npy' % (tid, K))).ravel()
    X_test[i] = vq

In [11]:
n_components = 100
coder = pmf.PoissonMF(n_components=n_components, random_state=98765, verbose=True)

In [12]:
coder.fit(X)


	After ITERATION: 39	Objective: 10723270.57	Old objective: 10718086.16	Improvement: 0.00048
pmf.py:164: RuntimeWarning: invalid value encountered in double_scalars
  improvement = (bound - old_bd) / abs(old_bd)
Out[12]:
PoissonMF(max_iter=100, n_components=100, random_state=98765, smoothness=100,
     tol=0.0005, verbose=True)

In [13]:
# randomly plot 30 "topics"
indices = np.random.choice(n_components, size=30, replace=False)
figure(figsize=(45, 15))
for i in xrange(30):
    subplot(10, 3, i+1)
    topic = coder.Eb[indices[i]].copy()
    # properly normalize the BoT dimensions for visualization purposes
    topic[K:] /= topic[K:].max()
    topic[K:] *= topic[:K].max()
    bar(np.arange(D), topic)
    axvline(x=K, color='red')
    title('Component #%d' % indices[i])
#savefig('dict.eps')



In [14]:
tagger = pmf.PoissonMF(n_components=n_components, random_state=98765, verbose=True)

In [15]:
tagger.set_components(coder.gamma_b[:, :K], coder.rho_b)


Out[15]:
PoissonMF(max_iter=100, n_components=100, random_state=98765, smoothness=100,
     tol=0.0005, verbose=True)

In [16]:
Et = tagger.transform(X_test)


	After ITERATION: 18	Objective: 3308281.25	Old objective: 3306843.06	Improvement: 0.00043

In [17]:
Et /= Et.sum(axis=1, keepdims=True)

tags_predicted = Et.dot(coder.Eb[:, K:])
print tags_predicted.min(), tags_predicted.max()

div_factor = 3
tags_predicted = tags_predicted - div_factor * np.mean(tags_predicted, axis=0)


0.000949887065626 1.37936269595

In [18]:
predictat = 20
tags_true_binary = (y_test > 0)

print_out_metrics(tags_true_binary, tags_predicted, predictat)


Precision = 0.111 (0.007)
Recall = 0.108 (0.006)
F-score = 0.109
AROC = 0.645 (0.005)
AP = 0.098 (0.005)

Stochastic inference on 10K subset


In [19]:
n_components = 100
online_coder = pmf.OnlinePoissonMF(n_components=n_components, batch_size=500, n_pass=1, 
                                   random_state=98765, verbose=True)

In [20]:
online_coder.fit(X, est_total=len(train_tracks))


Iteration 0: passing through the data...
	Minibatch 1:
	After ITERATION: 99	Objective: -478860.33	Old objective: -479135.19	Improvement: 0.00057
	Minibatch 2:
	After ITERATION: 72	Objective: 81908.86	Old objective: 81869.52	Improvement: 0.00048
	Minibatch 3:
	After ITERATION: 41	Objective: 215431.73	Old objective: 215327.13	Improvement: 0.00049
	Minibatch 4:
	After ITERATION: 33	Objective: 314231.33	Old objective: 314081.70	Improvement: 0.00048
	Minibatch 5:
	After ITERATION: 31	Objective: 319391.74	Old objective: 319233.27	Improvement: 0.00050
	Minibatch 6:
	After ITERATION: 31	Objective: 308588.24	Old objective: 308445.50	Improvement: 0.00046
	Minibatch 7:
	After ITERATION: 29	Objective: 332127.23	Old objective: 331977.98	Improvement: 0.00045
	Minibatch 8:
	After ITERATION: 27	Objective: 332379.38	Old objective: 332222.46	Improvement: 0.00047
	Minibatch 9:
	After ITERATION: 27	Objective: 342570.22	Old objective: 342418.70	Improvement: 0.00044
	Minibatch 10:
	After ITERATION: 26	Objective: 333090.29	Old objective: 332924.65	Improvement: 0.00050
	Minibatch 11:
	After ITERATION: 25	Objective: 397977.49	Old objective: 397796.71	Improvement: 0.00045
	Minibatch 12:
	After ITERATION: 24	Objective: 414687.27	Old objective: 414486.38	Improvement: 0.00048
	Minibatch 13:
	After ITERATION: 24	Objective: 387744.02	Old objective: 387560.34	Improvement: 0.00047
	Minibatch 14:
	After ITERATION: 24	Objective: 394930.86	Old objective: 394760.36	Improvement: 0.00043
	Minibatch 15:
	After ITERATION: 24	Objective: 385362.00	Old objective: 385183.56	Improvement: 0.00046
	Minibatch 16:
	After ITERATION: 22	Objective: 442326.58	Old objective: 442115.38	Improvement: 0.00048
	Minibatch 17:
	After ITERATION: 23	Objective: 416617.39	Old objective: 416430.74	Improvement: 0.00045
	Minibatch 18:
	After ITERATION: 21	Objective: 471153.75	Old objective: 470925.10	Improvement: 0.00049
	Minibatch 19:
	After ITERATION: 21	Objective: 445595.57	Old objective: 445377.52	Improvement: 0.00049
	Minibatch 20:
	After ITERATION: 21	Objective: 469278.29	Old objective: 469057.28	Improvement: 0.00047
Out[20]:
OnlinePoissonMF(batch_size=500, max_iter=100, n_components=100, n_pass=1,
        random_state=98765, shuffle=True, smoothness=100, tol=0.0005,
        verbose=True)

In [21]:
plot(online_coder.bound)
pass



In [22]:
ents = np.zeros((n_components, ))

for k in xrange(n_components):
    ents[k] = scipy.stats.entropy(online_coder.Eb[k])

In [23]:
idx = np.argsort(-ents)

In [24]:
plot(ents[idx], '-o')
pass



In [25]:
tagger = pmf.PoissonMF(n_components=n_components, random_state=98765, verbose=True)

In [26]:
tagger.set_components(online_coder.gamma_b[:, :K], online_coder.rho_b[:, :K])


Out[26]:
PoissonMF(max_iter=100, n_components=100, random_state=98765, smoothness=100,
     tol=0.0005, verbose=True)

In [27]:
Et = tagger.transform(X_test)


	After ITERATION: 20	Objective: 3067502.43	Old objective: 3066061.10	Improvement: 0.00047

In [28]:
Et /= Et.sum(axis=1, keepdims=True)

tags_predicted = Et.dot(online_coder.Eb[:, K:])
n_samples, n_tags = tags_predicted.shape

print tags_predicted.min(), tags_predicted.max()

div_factor = 3
tags_predicted = tags_predicted - div_factor * np.mean(tags_predicted, axis=0)


5.36941053794e-05 1.09076726478

In [29]:
predictat = 20
tags_true_binary = (y_test > 0)

print_out_metrics(tags_true_binary, tags_predicted, predictat)


Precision = 0.116 (0.007)
Recall = 0.131 (0.007)
F-score = 0.123
AROC = 0.687 (0.005)
AP = 0.110 (0.006)

Stochastic inference on the full set


In [30]:
# very naive implementation of out-of-core fit for stochastic PMF
def ooc_fit(obj, train_tracks, K, n_feats):
    n_samples = len(train_tracks)
    obj._scale = float(n_samples) / obj.batch_size
    obj._init_components(n_feats)
    obj.bound = list()

    for count in xrange(obj.n_pass):
        print 'Iteration %d: passing through the data...' % count
        indices = np.arange(n_samples)
        if obj.shuffle:
            np.random.shuffle(indices)
        for (i, istart) in enumerate(xrange(0, n_samples,
                                            obj.batch_size), 1):
            print 'Mini-batch %d:' % i
            iend = min(istart + obj.batch_size, n_samples)
            obj.set_learning_rate(iter=i)
            mini_batch = np.zeros((iend - istart, n_feats))
            for s in xrange(iend - istart):
                tid = train_tracks[indices[istart + s]]
                #print '\tRead in track: %s' % tid
                tdir = os.path.join('vq_hist', '/'.join(tid[2:5]))
                vq = np.load(os.path.join(tdir, '%s_K%d.npy' % (tid, K))).ravel()
                bot = np.load(os.path.join(tdir, '%s_BoT.npy' % tid))
                bot[bot > 0] = 1               
                mini_batch[s] = np.hstack((vq, bot))
            obj.partial_fit(mini_batch)
            obj.bound.append(obj._stoch_bound(mini_batch))
    return obj

In [31]:
n_components = 100
batch_size = 1000
online_coder_full = pmf.OnlinePoissonMF(n_components=n_components, batch_size=batch_size, n_pass=1, 
                                        random_state=98765, verbose=True)

In [32]:
online_coder_full = ooc_fit(online_coder_full, train_tracks, K, D)


Iteration 0: passing through the data...
Mini-batch 1:
	After ITERATION: 99	Objective: -838968.48	Old objective: -839520.23	Improvement: 0.00066
Mini-batch 2:
	After ITERATION: 55	Objective: 330451.59	Old objective: 330293.21	Improvement: 0.00048
Mini-batch 3:
	After ITERATION: 38	Objective: 553125.94	Old objective: 552867.14	Improvement: 0.00047
Mini-batch 4:
	After ITERATION: 33	Objective: 635871.16	Old objective: 635556.16	Improvement: 0.00050
Mini-batch 5:
	After ITERATION: 30	Objective: 695878.24	Old objective: 695546.55	Improvement: 0.00048
Mini-batch 6:
	After ITERATION: 28	Objective: 768466.09	Old objective: 768087.58	Improvement: 0.00049
Mini-batch 7:
	After ITERATION: 28	Objective: 739469.91	Old objective: 739118.06	Improvement: 0.00048
Mini-batch 8:
	After ITERATION: 26	Objective: 836343.78	Old objective: 835954.64	Improvement: 0.00047
Mini-batch 9:
	After ITERATION: 26	Objective: 852619.10	Old objective: 852227.48	Improvement: 0.00046
Mini-batch 10:
	After ITERATION: 25	Objective: 819482.01	Old objective: 819082.19	Improvement: 0.00049
Mini-batch 11:
	After ITERATION: 24	Objective: 845208.45	Old objective: 844814.10	Improvement: 0.00047
Mini-batch 12:
	After ITERATION: 24	Objective: 845044.81	Old objective: 844642.22	Improvement: 0.00048
Mini-batch 13:
	After ITERATION: 24	Objective: 816059.30	Old objective: 815666.91	Improvement: 0.00048
Mini-batch 14:
	After ITERATION: 24	Objective: 794065.94	Old objective: 793680.55	Improvement: 0.00049
Mini-batch 15:
	After ITERATION: 22	Objective: 974278.44	Old objective: 973796.11	Improvement: 0.00050
Mini-batch 16:
	After ITERATION: 23	Objective: 896549.23	Old objective: 896143.46	Improvement: 0.00045
Mini-batch 17:
	After ITERATION: 22	Objective: 863382.90	Old objective: 862961.57	Improvement: 0.00049
Mini-batch 18:
	After ITERATION: 23	Objective: 882706.00	Old objective: 882306.14	Improvement: 0.00045
Mini-batch 19:
	After ITERATION: 22	Objective: 891975.51	Old objective: 891550.33	Improvement: 0.00048
Mini-batch 20:
	After ITERATION: 21	Objective: 924968.90	Old objective: 924517.78	Improvement: 0.00049
Mini-batch 21:
	After ITERATION: 21	Objective: 948423.94	Old objective: 947972.97	Improvement: 0.00048
Mini-batch 22:
	After ITERATION: 21	Objective: 855681.70	Old objective: 855254.13	Improvement: 0.00050
Mini-batch 23:
	After ITERATION: 20	Objective: 979569.87	Old objective: 979086.64	Improvement: 0.00049
Mini-batch 24:
	After ITERATION: 21	Objective: 959566.25	Old objective: 959114.87	Improvement: 0.00047
Mini-batch 25:
	After ITERATION: 20	Objective: 966727.63	Old objective: 966257.28	Improvement: 0.00049
Mini-batch 26:
	After ITERATION: 21	Objective: 924176.66	Old objective: 923751.42	Improvement: 0.00046
Mini-batch 27:
	After ITERATION: 20	Objective: 955808.82	Old objective: 955350.94	Improvement: 0.00048
Mini-batch 28:
	After ITERATION: 20	Objective: 933417.55	Old objective: 932984.50	Improvement: 0.00046
Mini-batch 29:
	After ITERATION: 20	Objective: 934965.92	Old objective: 934510.41	Improvement: 0.00049
Mini-batch 30:
	After ITERATION: 20	Objective: 923400.92	Old objective: 922950.94	Improvement: 0.00049
Mini-batch 31:
	After ITERATION: 19	Objective: 1011637.22	Old objective: 1011153.42	Improvement: 0.00048
Mini-batch 32:
	After ITERATION: 20	Objective: 919927.35	Old objective: 919490.85	Improvement: 0.00047
Mini-batch 33:
	After ITERATION: 19	Objective: 983848.50	Old objective: 983359.24	Improvement: 0.00050
Mini-batch 34:
	After ITERATION: 19	Objective: 1011851.79	Old objective: 1011374.01	Improvement: 0.00047
Mini-batch 35:
	After ITERATION: 19	Objective: 943447.83	Old objective: 942977.35	Improvement: 0.00050
Mini-batch 36:
	After ITERATION: 19	Objective: 950197.67	Old objective: 949724.08	Improvement: 0.00050
Mini-batch 37:
	After ITERATION: 19	Objective: 992837.84	Old objective: 992395.29	Improvement: 0.00045
Mini-batch 38:
	After ITERATION: 19	Objective: 981298.24	Old objective: 980827.19	Improvement: 0.00048
Mini-batch 39:
	After ITERATION: 19	Objective: 994155.85	Old objective: 993681.70	Improvement: 0.00048
Mini-batch 40:
	After ITERATION: 19	Objective: 944250.43	Old objective: 943814.25	Improvement: 0.00046
Mini-batch 41:
	After ITERATION: 19	Objective: 952229.46	Old objective: 951816.84	Improvement: 0.00043
Mini-batch 42:
	After ITERATION: 18	Objective: 991395.34	Old objective: 990905.65	Improvement: 0.00049
Mini-batch 43:
	After ITERATION: 19	Objective: 979112.73	Old objective: 978678.59	Improvement: 0.00044
Mini-batch 44:
	After ITERATION: 18	Objective: 1021684.05	Old objective: 1021176.08	Improvement: 0.00050
Mini-batch 45:
	After ITERATION: 19	Objective: 964037.28	Old objective: 963590.83	Improvement: 0.00046
Mini-batch 46:
	After ITERATION: 19	Objective: 996080.71	Old objective: 995649.74	Improvement: 0.00043
Mini-batch 47:
	After ITERATION: 18	Objective: 1007201.75	Old objective: 1006727.54	Improvement: 0.00047
Mini-batch 48:
	After ITERATION: 19	Objective: 963821.48	Old objective: 963378.24	Improvement: 0.00046
Mini-batch 49:
	After ITERATION: 18	Objective: 977100.85	Old objective: 976616.29	Improvement: 0.00050
Mini-batch 50:
	After ITERATION: 18	Objective: 1037832.34	Old objective: 1037336.13	Improvement: 0.00048
Mini-batch 51:
	After ITERATION: 18	Objective: 1019284.29	Old objective: 1018816.05	Improvement: 0.00046
Mini-batch 52:
	After ITERATION: 19	Objective: 957454.85	Old objective: 957031.01	Improvement: 0.00044
Mini-batch 53:
	After ITERATION: 19	Objective: 932207.71	Old objective: 931787.92	Improvement: 0.00045
Mini-batch 54:
	After ITERATION: 18	Objective: 1043985.60	Old objective: 1043509.42	Improvement: 0.00046
Mini-batch 55:
	After ITERATION: 17	Objective: 1054681.95	Old objective: 1054169.84	Improvement: 0.00049
Mini-batch 56:
	After ITERATION: 18	Objective: 947185.90	Old objective: 946717.33	Improvement: 0.00049
Mini-batch 57:
	After ITERATION: 18	Objective: 962078.55	Old objective: 961625.79	Improvement: 0.00047
Mini-batch 58:
	After ITERATION: 18	Objective: 954449.56	Old objective: 953988.60	Improvement: 0.00048
Mini-batch 59:
	After ITERATION: 18	Objective: 1033015.34	Old objective: 1032560.56	Improvement: 0.00044
Mini-batch 60:
	After ITERATION: 18	Objective: 950030.86	Old objective: 949565.39	Improvement: 0.00049
Mini-batch 61:
	After ITERATION: 18	Objective: 1006097.61	Old objective: 1005648.49	Improvement: 0.00045
Mini-batch 62:
	After ITERATION: 18	Objective: 1010524.60	Old objective: 1010063.96	Improvement: 0.00046
Mini-batch 63:
	After ITERATION: 18	Objective: 960181.94	Old objective: 959740.15	Improvement: 0.00046
Mini-batch 64:
	After ITERATION: 18	Objective: 1005279.89	Old objective: 1004814.24	Improvement: 0.00046
Mini-batch 65:
	After ITERATION: 18	Objective: 1006458.92	Old objective: 1006024.37	Improvement: 0.00043
Mini-batch 66:
	After ITERATION: 18	Objective: 1025903.07	Old objective: 1025452.39	Improvement: 0.00044
Mini-batch 67:
	After ITERATION: 18	Objective: 984433.06	Old objective: 983998.63	Improvement: 0.00044
Mini-batch 68:
	After ITERATION: 17	Objective: 1024722.72	Old objective: 1024232.46	Improvement: 0.00048
Mini-batch 69:
	After ITERATION: 17	Objective: 1049168.83	Old objective: 1048676.28	Improvement: 0.00047
Mini-batch 70:
	After ITERATION: 18	Objective: 939387.34	Old objective: 938955.50	Improvement: 0.00046
Mini-batch 71:
	After ITERATION: 18	Objective: 1017890.59	Old objective: 1017460.27	Improvement: 0.00042
Mini-batch 72:
	After ITERATION: 17	Objective: 1083556.84	Old objective: 1083076.20	Improvement: 0.00044
Mini-batch 73:
	After ITERATION: 17	Objective: 1003922.28	Old objective: 1003433.21	Improvement: 0.00049
Mini-batch 74:
	After ITERATION: 17	Objective: 1043622.76	Old objective: 1043121.90	Improvement: 0.00048
Mini-batch 75:
	After ITERATION: 18	Objective: 974777.54	Old objective: 974338.72	Improvement: 0.00045
Mini-batch 76:
	After ITERATION: 17	Objective: 1018929.83	Old objective: 1018472.21	Improvement: 0.00045
Mini-batch 77:
	After ITERATION: 17	Objective: 977949.65	Old objective: 977465.20	Improvement: 0.00050
Mini-batch 78:
	After ITERATION: 17	Objective: 1066141.66	Old objective: 1065660.94	Improvement: 0.00045
Mini-batch 79:
	After ITERATION: 18	Objective: 969007.97	Old objective: 968573.76	Improvement: 0.00045
Mini-batch 80:
	After ITERATION: 18	Objective: 990091.70	Old objective: 989666.39	Improvement: 0.00043
Mini-batch 81:
	After ITERATION: 17	Objective: 1012554.69	Old objective: 1012060.69	Improvement: 0.00049
Mini-batch 82:
	After ITERATION: 17	Objective: 1075548.13	Old objective: 1075070.57	Improvement: 0.00044
Mini-batch 83:
	After ITERATION: 17	Objective: 1013357.05	Old objective: 1012917.77	Improvement: 0.00043
Mini-batch 84:
	After ITERATION: 17	Objective: 951811.85	Old objective: 951341.39	Improvement: 0.00049
Mini-batch 85:
	After ITERATION: 17	Objective: 1039373.30	Old objective: 1038916.34	Improvement: 0.00044
Mini-batch 86:
	After ITERATION: 17	Objective: 1041026.72	Old objective: 1040543.29	Improvement: 0.00046
Mini-batch 87:
	After ITERATION: 17	Objective: 981806.83	Old objective: 981337.39	Improvement: 0.00048
Mini-batch 88:
	After ITERATION: 17	Objective: 984409.75	Old objective: 983922.09	Improvement: 0.00050
Mini-batch 89:
	After ITERATION: 17	Objective: 974591.89	Old objective: 974136.58	Improvement: 0.00047
Mini-batch 90:
	After ITERATION: 17	Objective: 1042582.16	Old objective: 1042111.95	Improvement: 0.00045
Mini-batch 91:
	After ITERATION: 17	Objective: 1072026.12	Old objective: 1071567.77	Improvement: 0.00043
Mini-batch 92:
	After ITERATION: 16	Objective: 1040577.82	Old objective: 1040063.57	Improvement: 0.00049
Mini-batch 93:
	After ITERATION: 17	Objective: 974010.32	Old objective: 973526.75	Improvement: 0.00050
Mini-batch 94:
	After ITERATION: 17	Objective: 1070298.98	Old objective: 1069809.17	Improvement: 0.00046
Mini-batch 95:
	After ITERATION: 17	Objective: 919223.29	Old objective: 918769.84	Improvement: 0.00049
Mini-batch 96:
	After ITERATION: 17	Objective: 1029070.82	Old objective: 1028594.58	Improvement: 0.00046
Mini-batch 97:
	After ITERATION: 17	Objective: 1017395.00	Old objective: 1016947.57	Improvement: 0.00044
Mini-batch 98:
	After ITERATION: 17	Objective: 1017731.10	Old objective: 1017278.11	Improvement: 0.00045
Mini-batch 99:
	After ITERATION: 17	Objective: 972302.05	Old objective: 971851.16	Improvement: 0.00046
Mini-batch 100:
	After ITERATION: 17	Objective: 1032231.04	Old objective: 1031776.26	Improvement: 0.00044
Mini-batch 101:
	After ITERATION: 17	Objective: 992875.16	Old objective: 992428.54	Improvement: 0.00045
Mini-batch 102:
	After ITERATION: 16	Objective: 1071424.25	Old objective: 1070903.82	Improvement: 0.00049
Mini-batch 103:
	After ITERATION: 16	Objective: 1068643.98	Old objective: 1068111.96	Improvement: 0.00050
Mini-batch 104:
	After ITERATION: 16	Objective: 1076202.58	Old objective: 1075678.17	Improvement: 0.00049
Mini-batch 105:
	After ITERATION: 17	Objective: 1052727.79	Old objective: 1052275.29	Improvement: 0.00043
Mini-batch 106:
	After ITERATION: 16	Objective: 1077327.28	Old objective: 1076794.34	Improvement: 0.00049
Mini-batch 107:
	After ITERATION: 17	Objective: 1035360.77	Old objective: 1034917.38	Improvement: 0.00043
Mini-batch 108:
	After ITERATION: 16	Objective: 1051049.96	Old objective: 1050535.16	Improvement: 0.00049
Mini-batch 109:
	After ITERATION: 16	Objective: 1089714.15	Old objective: 1089189.53	Improvement: 0.00048
Mini-batch 110:
	After ITERATION: 16	Objective: 1056248.04	Old objective: 1055756.68	Improvement: 0.00047
Mini-batch 111:
	After ITERATION: 16	Objective: 1120971.59	Old objective: 1120436.46	Improvement: 0.00048
Mini-batch 112:
	After ITERATION: 16	Objective: 1063079.05	Old objective: 1062564.31	Improvement: 0.00048
Mini-batch 113:
	After ITERATION: 17	Objective: 988163.42	Old objective: 987743.95	Improvement: 0.00042
Mini-batch 114:
	After ITERATION: 17	Objective: 1013004.93	Old objective: 1012561.58	Improvement: 0.00044
Mini-batch 115:
	After ITERATION: 16	Objective: 1082110.48	Old objective: 1081606.30	Improvement: 0.00047
Mini-batch 116:
	After ITERATION: 17	Objective: 1005101.06	Old objective: 1004662.91	Improvement: 0.00044
Mini-batch 117:
	After ITERATION: 17	Objective: 1003218.58	Old objective: 1002777.58	Improvement: 0.00044
Mini-batch 118:
	After ITERATION: 16	Objective: 1115720.47	Old objective: 1115202.88	Improvement: 0.00046
Mini-batch 119:
	After ITERATION: 16	Objective: 1051355.74	Old objective: 1050854.45	Improvement: 0.00048
Mini-batch 120:
	After ITERATION: 16	Objective: 1086837.69	Old objective: 1086318.34	Improvement: 0.00048
Mini-batch 121:
	After ITERATION: 16	Objective: 1061516.71	Old objective: 1061009.79	Improvement: 0.00048
Mini-batch 122:
	After ITERATION: 17	Objective: 988589.83	Old objective: 988164.22	Improvement: 0.00043
Mini-batch 123:
	After ITERATION: 16	Objective: 1025563.35	Old objective: 1025072.77	Improvement: 0.00048
Mini-batch 124:
	After ITERATION: 17	Objective: 1067431.06	Old objective: 1066969.37	Improvement: 0.00043
Mini-batch 125:
	After ITERATION: 16	Objective: 1050025.33	Old objective: 1049537.93	Improvement: 0.00046
Mini-batch 126:
	After ITERATION: 16	Objective: 1019439.84	Old objective: 1018956.99	Improvement: 0.00047
Mini-batch 127:
	After ITERATION: 16	Objective: 1045361.87	Old objective: 1044845.38	Improvement: 0.00049
Mini-batch 128:
	After ITERATION: 16	Objective: 1056190.13	Old objective: 1055689.20	Improvement: 0.00047
Mini-batch 129:
	After ITERATION: 16	Objective: 1102463.37	Old objective: 1101964.14	Improvement: 0.00045
Mini-batch 130:
	After ITERATION: 16	Objective: 1051361.05	Old objective: 1050865.13	Improvement: 0.00047
Mini-batch 131:
	After ITERATION: 16	Objective: 1060032.77	Old objective: 1059545.76	Improvement: 0.00046
Mini-batch 132:
	After ITERATION: 16	Objective: 1043248.71	Old objective: 1042758.84	Improvement: 0.00047
Mini-batch 133:
	After ITERATION: 16	Objective: 1012186.66	Old objective: 1011731.13	Improvement: 0.00045
Mini-batch 134:
	After ITERATION: 16	Objective: 1023425.71	Old objective: 1022977.10	Improvement: 0.00044
Mini-batch 135:
	After ITERATION: 17	Objective: 1012973.09	Old objective: 1012536.65	Improvement: 0.00043
Mini-batch 136:
	After ITERATION: 17	Objective: 933590.39	Old objective: 933189.01	Improvement: 0.00043
Mini-batch 137:
	After ITERATION: 16	Objective: 1060170.38	Old objective: 1059658.45	Improvement: 0.00048
Mini-batch 138:
	After ITERATION: 17	Objective: 1000358.72	Old objective: 999924.72	Improvement: 0.00043
Mini-batch 139:
	After ITERATION: 17	Objective: 1010732.43	Old objective: 1010312.39	Improvement: 0.00042
Mini-batch 140:
	After ITERATION: 16	Objective: 1025983.25	Old objective: 1025508.92	Improvement: 0.00046
Mini-batch 141:
	After ITERATION: 16	Objective: 1090344.03	Old objective: 1089878.39	Improvement: 0.00043
Mini-batch 142:
	After ITERATION: 16	Objective: 1041233.29	Old objective: 1040722.70	Improvement: 0.00049
Mini-batch 143:
	After ITERATION: 16	Objective: 1020799.80	Old objective: 1020322.90	Improvement: 0.00047
Mini-batch 144:
	After ITERATION: 16	Objective: 1093435.88	Old objective: 1092948.65	Improvement: 0.00045
Mini-batch 145:
	After ITERATION: 16	Objective: 1012263.74	Old objective: 1011792.12	Improvement: 0.00047
Mini-batch 146:
	After ITERATION: 16	Objective: 1055632.66	Old objective: 1055154.68	Improvement: 0.00045
Mini-batch 147:
	After ITERATION: 15	Objective: 1147956.81	Old objective: 1147393.11	Improvement: 0.00049
Mini-batch 148:
	After ITERATION: 16	Objective: 1092930.99	Old objective: 1092449.68	Improvement: 0.00044
Mini-batch 149:
	After ITERATION: 16	Objective: 1055100.46	Old objective: 1054621.33	Improvement: 0.00045
Mini-batch 150:
	After ITERATION: 16	Objective: 1065273.87	Old objective: 1064767.40	Improvement: 0.00048
Mini-batch 151:
	After ITERATION: 16	Objective: 1066387.52	Old objective: 1065905.20	Improvement: 0.00045
Mini-batch 152:
	After ITERATION: 16	Objective: 1009737.48	Old objective: 1009272.22	Improvement: 0.00046
Mini-batch 153:
	After ITERATION: 16	Objective: 1096544.91	Old objective: 1096056.93	Improvement: 0.00045
Mini-batch 154:
	After ITERATION: 16	Objective: 967781.76	Old objective: 967302.45	Improvement: 0.00050
Mini-batch 155:
	After ITERATION: 16	Objective: 1123083.12	Old objective: 1122600.07	Improvement: 0.00043
Mini-batch 156:
	After ITERATION: 16	Objective: 1033903.96	Old objective: 1033433.67	Improvement: 0.00046
Mini-batch 157:
	After ITERATION: 16	Objective: 1094916.06	Old objective: 1094442.77	Improvement: 0.00043
Mini-batch 158:
	After ITERATION: 16	Objective: 1078242.74	Old objective: 1077763.44	Improvement: 0.00044
Mini-batch 159:
	After ITERATION: 16	Objective: 1077030.00	Old objective: 1076526.90	Improvement: 0.00047
Mini-batch 160:
	After ITERATION: 16	Objective: 1070248.44	Old objective: 1069768.57	Improvement: 0.00045
Mini-batch 161:
	After ITERATION: 16	Objective: 1076878.16	Old objective: 1076417.44	Improvement: 0.00043
Mini-batch 162:
	After ITERATION: 16	Objective: 1049686.30	Old objective: 1049211.61	Improvement: 0.00045
Mini-batch 163:
	After ITERATION: 16	Objective: 1033665.03	Old objective: 1033190.74	Improvement: 0.00046
Mini-batch 164:
	After ITERATION: 16	Objective: 1043869.00	Old objective: 1043427.98	Improvement: 0.00042
Mini-batch 165:
	After ITERATION: 16	Objective: 1078463.24	Old objective: 1077994.44	Improvement: 0.00043
Mini-batch 166:
	After ITERATION: 16	Objective: 1030744.67	Old objective: 1030258.99	Improvement: 0.00047
Mini-batch 167:
	After ITERATION: 16	Objective: 1053964.42	Old objective: 1053488.10	Improvement: 0.00045
Mini-batch 168:
	After ITERATION: 16	Objective: 1082187.87	Old objective: 1081727.80	Improvement: 0.00043
Mini-batch 169:
	After ITERATION: 16	Objective: 1103763.62	Old objective: 1103293.68	Improvement: 0.00043
Mini-batch 170:
	After ITERATION: 16	Objective: 1023654.26	Old objective: 1023202.09	Improvement: 0.00044
Mini-batch 171:
	After ITERATION: 16	Objective: 1011081.77	Old objective: 1010593.43	Improvement: 0.00048
Mini-batch 172:
	After ITERATION: 16	Objective: 1054799.34	Old objective: 1054313.91	Improvement: 0.00046
Mini-batch 173:
	After ITERATION: 16	Objective: 1079780.98	Old objective: 1079305.87	Improvement: 0.00044
Mini-batch 174:
	After ITERATION: 16	Objective: 1055849.95	Old objective: 1055402.91	Improvement: 0.00042
Mini-batch 175:
	After ITERATION: 15	Objective: 1083305.30	Old objective: 1082774.12	Improvement: 0.00049
Mini-batch 176:
	After ITERATION: 15	Objective: 1097658.41	Old objective: 1097130.46	Improvement: 0.00048
Mini-batch 177:
	After ITERATION: 16	Objective: 979952.04	Old objective: 979478.57	Improvement: 0.00048
Mini-batch 178:
	After ITERATION: 16	Objective: 1069556.34	Old objective: 1069066.54	Improvement: 0.00046
Mini-batch 179:
	After ITERATION: 16	Objective: 1066575.02	Old objective: 1066083.93	Improvement: 0.00046
Mini-batch 180:
	After ITERATION: 16	Objective: 1007094.48	Old objective: 1006640.32	Improvement: 0.00045
Mini-batch 181:
	After ITERATION: 16	Objective: 1056688.95	Old objective: 1056235.20	Improvement: 0.00043
Mini-batch 182:
	After ITERATION: 16	Objective: 1070172.36	Old objective: 1069703.07	Improvement: 0.00044
Mini-batch 183:
	After ITERATION: 15	Objective: 1104483.78	Old objective: 1103975.61	Improvement: 0.00046
Mini-batch 184:
	After ITERATION: 16	Objective: 1043102.31	Old objective: 1042610.55	Improvement: 0.00047
Mini-batch 185:
	After ITERATION: 16	Objective: 1019335.46	Old objective: 1018893.88	Improvement: 0.00043
Mini-batch 186:
	After ITERATION: 16	Objective: 1055281.72	Old objective: 1054809.16	Improvement: 0.00045
Mini-batch 187:
	After ITERATION: 16	Objective: 1067702.95	Old objective: 1067224.44	Improvement: 0.00045
Mini-batch 188:
	After ITERATION: 15	Objective: 1047741.34	Old objective: 1047221.12	Improvement: 0.00050
Mini-batch 189:
	After ITERATION: 16	Objective: 1075517.60	Old objective: 1075055.43	Improvement: 0.00043
Mini-batch 190:
	After ITERATION: 15	Objective: 1122322.28	Old objective: 1121793.34	Improvement: 0.00047
Mini-batch 191:
	After ITERATION: 15	Objective: 1064793.83	Old objective: 1064262.08	Improvement: 0.00050
Mini-batch 192:
	After ITERATION: 16	Objective: 1077074.17	Old objective: 1076617.63	Improvement: 0.00042
Mini-batch 193:
	After ITERATION: 15	Objective: 1036406.98	Old objective: 1035905.83	Improvement: 0.00048
Mini-batch 194:
	After ITERATION: 16	Objective: 1028260.18	Old objective: 1027822.58	Improvement: 0.00043
Mini-batch 195:
	After ITERATION: 16	Objective: 1024116.10	Old objective: 1023657.15	Improvement: 0.00045
Mini-batch 196:
	After ITERATION: 15	Objective: 1097937.37	Old objective: 1097407.99	Improvement: 0.00048
Mini-batch 197:
	After ITERATION: 15	Objective: 1064198.53	Old objective: 1063666.83	Improvement: 0.00050
Mini-batch 198:
	After ITERATION: 15	Objective: 1068370.68	Old objective: 1067851.59	Improvement: 0.00049
Mini-batch 199:
	After ITERATION: 15	Objective: 1089779.92	Old objective: 1089249.85	Improvement: 0.00049
Mini-batch 200:
	After ITERATION: 16	Objective: 1032170.17	Old objective: 1031713.00	Improvement: 0.00044
Mini-batch 201:
	After ITERATION: 15	Objective: 1091191.64	Old objective: 1090665.49	Improvement: 0.00048
Mini-batch 202:
	After ITERATION: 15	Objective: 1099750.64	Old objective: 1099201.11	Improvement: 0.00050
Mini-batch 203:
	After ITERATION: 15	Objective: 1127486.35	Old objective: 1126933.62	Improvement: 0.00049
Mini-batch 204:
	After ITERATION: 15	Objective: 1091404.22	Old objective: 1090864.50	Improvement: 0.00049
Mini-batch 205:
	After ITERATION: 16	Objective: 1028463.59	Old objective: 1027991.65	Improvement: 0.00046
Mini-batch 206:
	After ITERATION: 15	Objective: 1134642.72	Old objective: 1134114.61	Improvement: 0.00047
Mini-batch 207:
	After ITERATION: 16	Objective: 1078184.39	Old objective: 1077719.02	Improvement: 0.00043
Mini-batch 208:
	After ITERATION: 15	Objective: 1065003.36	Old objective: 1064475.16	Improvement: 0.00050
Mini-batch 209:
	After ITERATION: 15	Objective: 1130336.20	Old objective: 1129794.52	Improvement: 0.00048
Mini-batch 210:
	After ITERATION: 15	Objective: 1079257.19	Old objective: 1078740.22	Improvement: 0.00048
Mini-batch 211:
	After ITERATION: 16	Objective: 1016802.41	Old objective: 1016337.03	Improvement: 0.00046
Mini-batch 212:
	After ITERATION: 15	Objective: 1119807.24	Old objective: 1119279.49	Improvement: 0.00047
Mini-batch 213:
	After ITERATION: 15	Objective: 1131064.12	Old objective: 1130509.74	Improvement: 0.00049
Mini-batch 214:
	After ITERATION: 15	Objective: 1129272.38	Old objective: 1128748.11	Improvement: 0.00046
Mini-batch 215:
	After ITERATION: 16	Objective: 1048826.75	Old objective: 1048373.14	Improvement: 0.00043
Mini-batch 216:
	After ITERATION: 16	Objective: 1055251.28	Old objective: 1054791.41	Improvement: 0.00044
Mini-batch 217:
	After ITERATION: 15	Objective: 1095407.38	Old objective: 1094874.57	Improvement: 0.00049
Mini-batch 218:
	After ITERATION: 16	Objective: 987980.79	Old objective: 987537.27	Improvement: 0.00045
Mini-batch 219:
	After ITERATION: 15	Objective: 1067688.94	Old objective: 1067189.11	Improvement: 0.00047
Mini-batch 220:
	After ITERATION: 15	Objective: 1087429.77	Old objective: 1086909.52	Improvement: 0.00048
Mini-batch 221:
	After ITERATION: 16	Objective: 1044031.75	Old objective: 1043571.85	Improvement: 0.00044
Mini-batch 222:
	After ITERATION: 15	Objective: 1139762.08	Old objective: 1139242.24	Improvement: 0.00046
Mini-batch 223:
	After ITERATION: 16	Objective: 1037340.25	Old objective: 1036880.51	Improvement: 0.00044
Mini-batch 224:
	After ITERATION: 15	Objective: 1061372.14	Old objective: 1060846.19	Improvement: 0.00050
Mini-batch 225:
	After ITERATION: 16	Objective: 1029232.93	Old objective: 1028796.00	Improvement: 0.00042
Mini-batch 226:
	After ITERATION: 16	Objective: 1035628.26	Old objective: 1035188.07	Improvement: 0.00043
Mini-batch 227:
	After ITERATION: 15	Objective: 1127003.89	Old objective: 1126489.78	Improvement: 0.00046
Mini-batch 228:
	After ITERATION: 15	Objective: 1175307.14	Old objective: 1174770.40	Improvement: 0.00046
Mini-batch 229:
	After ITERATION: 15	Objective: 1166183.38	Old objective: 1165668.62	Improvement: 0.00044
Mini-batch 230:
	After ITERATION: 16	Objective: 1055566.81	Old objective: 1055104.68	Improvement: 0.00044
Mini-batch 231:
	After ITERATION: 15	Objective: 1080103.93	Old objective: 1079585.99	Improvement: 0.00048
Mini-batch 232:
	After ITERATION: 16	Objective: 1023238.83	Old objective: 1022779.87	Improvement: 0.00045
Mini-batch 233:
	After ITERATION: 15	Objective: 1149896.17	Old objective: 1149367.97	Improvement: 0.00046
Mini-batch 234:
	After ITERATION: 15	Objective: 1085351.84	Old objective: 1084843.20	Improvement: 0.00047
Mini-batch 235:
	After ITERATION: 16	Objective: 1097929.15	Old objective: 1097462.58	Improvement: 0.00043
Mini-batch 236:
	After ITERATION: 15	Objective: 1106732.98	Old objective: 1106203.83	Improvement: 0.00048
Mini-batch 237:
	After ITERATION: 15	Objective: 1093144.57	Old objective: 1092636.85	Improvement: 0.00046
Mini-batch 238:
	After ITERATION: 15	Objective: 1052911.72	Old objective: 1052402.43	Improvement: 0.00048
Mini-batch 239:
	After ITERATION: 15	Objective: 1127279.58	Old objective: 1126771.17	Improvement: 0.00045
Mini-batch 240:
	After ITERATION: 15	Objective: 1047605.76	Old objective: 1047101.00	Improvement: 0.00048
Mini-batch 241:
	After ITERATION: 15	Objective: 1105254.55	Old objective: 1104751.44	Improvement: 0.00046
Mini-batch 242:
	After ITERATION: 15	Objective: 1048030.79	Old objective: 1047511.15	Improvement: 0.00050
Mini-batch 243:
	After ITERATION: 15	Objective: 1074470.06	Old objective: 1073962.40	Improvement: 0.00047
Mini-batch 244:
	After ITERATION: 16	Objective: 1059822.60	Old objective: 1059351.91	Improvement: 0.00044
Mini-batch 245:
	After ITERATION: 15	Objective: 1089722.47	Old objective: 1089210.80	Improvement: 0.00047
Mini-batch 246:
	After ITERATION: 15	Objective: 1134588.01	Old objective: 1134067.32	Improvement: 0.00046
Mini-batch 247:
	After ITERATION: 15	Objective: 1052853.66	Old objective: 1052329.18	Improvement: 0.00050
Mini-batch 248:
	After ITERATION: 15	Objective: 1118512.10	Old objective: 1117994.96	Improvement: 0.00046
Mini-batch 249:
	After ITERATION: 15	Objective: 1024287.21	Old objective: 1023777.46	Improvement: 0.00050
Mini-batch 250:
	After ITERATION: 15	Objective: 1094616.91	Old objective: 1094123.15	Improvement: 0.00045
Mini-batch 251:
	After ITERATION: 15	Objective: 1053407.97	Old objective: 1052886.64	Improvement: 0.00050
Mini-batch 252:
	After ITERATION: 15	Objective: 1035696.69	Old objective: 1035202.16	Improvement: 0.00048
Mini-batch 253:
	After ITERATION: 15	Objective: 1162698.62	Old objective: 1162182.83	Improvement: 0.00044
Mini-batch 254:
	After ITERATION: 15	Objective: 1131789.54	Old objective: 1131281.23	Improvement: 0.00045
Mini-batch 255:
	After ITERATION: 15	Objective: 1089260.86	Old objective: 1088756.88	Improvement: 0.00046
Mini-batch 256:
	After ITERATION: 15	Objective: 1025664.28	Old objective: 1025167.56	Improvement: 0.00048
Mini-batch 257:
	After ITERATION: 15	Objective: 1151615.28	Old objective: 1151118.52	Improvement: 0.00043
Mini-batch 258:
	After ITERATION: 15	Objective: 1069403.77	Old objective: 1068899.49	Improvement: 0.00047
Mini-batch 259:
	After ITERATION: 15	Objective: 1072867.19	Old objective: 1072363.88	Improvement: 0.00047
Mini-batch 260:
	After ITERATION: 15	Objective: 1145507.86	Old objective: 1145021.19	Improvement: 0.00043
Mini-batch 261:
	After ITERATION: 16	Objective: 1012243.26	Old objective: 1011803.32	Improvement: 0.00043
Mini-batch 262:
	After ITERATION: 15	Objective: 1132702.92	Old objective: 1132196.73	Improvement: 0.00045
Mini-batch 263:
	After ITERATION: 15	Objective: 1088628.22	Old objective: 1088130.45	Improvement: 0.00046
Mini-batch 264:
	After ITERATION: 15	Objective: 1085389.36	Old objective: 1084887.79	Improvement: 0.00046
Mini-batch 265:
	After ITERATION: 15	Objective: 1052276.43	Old objective: 1051783.93	Improvement: 0.00047
Mini-batch 266:
	After ITERATION: 14	Objective: 1179647.71	Old objective: 1179060.01	Improvement: 0.00050
Mini-batch 267:
	After ITERATION: 14	Objective: 1175298.80	Old objective: 1174712.43	Improvement: 0.00050
Mini-batch 268:
	After ITERATION: 16	Objective: 1030178.26	Old objective: 1029732.70	Improvement: 0.00043
Mini-batch 269:
	After ITERATION: 15	Objective: 1030962.98	Old objective: 1030459.81	Improvement: 0.00049
Mini-batch 270:
	After ITERATION: 15	Objective: 1058562.19	Old objective: 1058058.28	Improvement: 0.00048
Mini-batch 271:
	After ITERATION: 15	Objective: 1027443.62	Old objective: 1026949.85	Improvement: 0.00048
Mini-batch 272:
	After ITERATION: 15	Objective: 1072814.88	Old objective: 1072297.86	Improvement: 0.00048
Mini-batch 273:
	After ITERATION: 15	Objective: 1074062.05	Old objective: 1073568.55	Improvement: 0.00046
Mini-batch 274:
	After ITERATION: 15	Objective: 1058107.49	Old objective: 1057602.51	Improvement: 0.00048
Mini-batch 275:
	After ITERATION: 15	Objective: 1106539.27	Old objective: 1106024.06	Improvement: 0.00047
Mini-batch 276:
	After ITERATION: 15	Objective: 1159843.60	Old objective: 1159335.14	Improvement: 0.00044
Mini-batch 277:
	After ITERATION: 15	Objective: 1065902.66	Old objective: 1065387.31	Improvement: 0.00048
Mini-batch 278:
	After ITERATION: 15	Objective: 989362.49	Old objective: 988870.68	Improvement: 0.00050
Mini-batch 279:
	After ITERATION: 15	Objective: 1076839.37	Old objective: 1076339.72	Improvement: 0.00046
Mini-batch 280:
	After ITERATION: 15	Objective: 1110278.49	Old objective: 1109775.50	Improvement: 0.00045
Mini-batch 281:
	After ITERATION: 15	Objective: 1042377.06	Old objective: 1041880.50	Improvement: 0.00048
Mini-batch 282:
	After ITERATION: 15	Objective: 1062631.77	Old objective: 1062125.96	Improvement: 0.00048
Mini-batch 283:
	After ITERATION: 15	Objective: 1075690.64	Old objective: 1075199.13	Improvement: 0.00046
Mini-batch 284:
	After ITERATION: 15	Objective: 1126961.23	Old objective: 1126442.03	Improvement: 0.00046
Mini-batch 285:
	After ITERATION: 15	Objective: 1122975.69	Old objective: 1122461.97	Improvement: 0.00046
Mini-batch 286:
	After ITERATION: 15	Objective: 1039876.28	Old objective: 1039405.20	Improvement: 0.00045
Mini-batch 287:
	After ITERATION: 15	Objective: 1116014.86	Old objective: 1115486.61	Improvement: 0.00047
Mini-batch 288:
	After ITERATION: 15	Objective: 1077953.25	Old objective: 1077473.43	Improvement: 0.00045
Mini-batch 289:
	After ITERATION: 15	Objective: 1089217.47	Old objective: 1088718.46	Improvement: 0.00046
Mini-batch 290:
	After ITERATION: 14	Objective: 1165825.84	Old objective: 1165249.45	Improvement: 0.00049
Mini-batch 291:
	After ITERATION: 15	Objective: 1106836.86	Old objective: 1106340.41	Improvement: 0.00045
Mini-batch 292:
	After ITERATION: 15	Objective: 1032560.01	Old objective: 1032077.71	Improvement: 0.00047
Mini-batch 293:
	After ITERATION: 15	Objective: 1027049.26	Old objective: 1026549.71	Improvement: 0.00049
Mini-batch 294:
	After ITERATION: 14	Objective: 1228125.42	Old objective: 1227538.27	Improvement: 0.00048
Mini-batch 295:
	After ITERATION: 15	Objective: 1097882.46	Old objective: 1097374.40	Improvement: 0.00046
Mini-batch 296:
	After ITERATION: 15	Objective: 1056043.15	Old objective: 1055546.46	Improvement: 0.00047
Mini-batch 297:
	After ITERATION: 15	Objective: 1076393.49	Old objective: 1075893.55	Improvement: 0.00046
Mini-batch 298:
	After ITERATION: 15	Objective: 1093885.09	Old objective: 1093368.91	Improvement: 0.00047
Mini-batch 299:
	After ITERATION: 15	Objective: 1017895.30	Old objective: 1017392.30	Improvement: 0.00049
Mini-batch 300:
	After ITERATION: 15	Objective: 1093496.13	Old objective: 1093015.35	Improvement: 0.00044
Mini-batch 301:
	After ITERATION: 15	Objective: 1038203.66	Old objective: 1037734.18	Improvement: 0.00045
Mini-batch 302:
	After ITERATION: 15	Objective: 1091164.84	Old objective: 1090694.54	Improvement: 0.00043
Mini-batch 303:
	After ITERATION: 15	Objective: 1099725.70	Old objective: 1099207.36	Improvement: 0.00047
Mini-batch 304:
	After ITERATION: 15	Objective: 1177362.43	Old objective: 1176845.71	Improvement: 0.00044
Mini-batch 305:
	After ITERATION: 15	Objective: 1139684.47	Old objective: 1139149.42	Improvement: 0.00047
Mini-batch 306:
	After ITERATION: 15	Objective: 1070106.28	Old objective: 1069603.90	Improvement: 0.00047
Mini-batch 307:
	After ITERATION: 15	Objective: 1127686.79	Old objective: 1127213.48	Improvement: 0.00042
Mini-batch 308:
	After ITERATION: 15	Objective: 1107884.48	Old objective: 1107392.22	Improvement: 0.00044
Mini-batch 309:
	After ITERATION: 15	Objective: 1094298.45	Old objective: 1093776.53	Improvement: 0.00048
Mini-batch 310:
	After ITERATION: 15	Objective: 1063033.31	Old objective: 1062522.29	Improvement: 0.00048
Mini-batch 311:
	After ITERATION: 15	Objective: 1105122.11	Old objective: 1104636.89	Improvement: 0.00044
Mini-batch 312:
	After ITERATION: 15	Objective: 1142106.67	Old objective: 1141616.15	Improvement: 0.00043
Mini-batch 313:
	After ITERATION: 15	Objective: 1108809.55	Old objective: 1108329.21	Improvement: 0.00043
Mini-batch 314:
	After ITERATION: 15	Objective: 1069988.46	Old objective: 1069468.85	Improvement: 0.00049
Mini-batch 315:
	After ITERATION: 15	Objective: 1097799.84	Old objective: 1097298.06	Improvement: 0.00046
Mini-batch 316:
	After ITERATION: 15	Objective: 1162605.14	Old objective: 1162114.87	Improvement: 0.00042
Mini-batch 317:
	After ITERATION: 15	Objective: 1109930.94	Old objective: 1109448.15	Improvement: 0.00044
Mini-batch 318:
	After ITERATION: 15	Objective: 1038812.48	Old objective: 1038346.80	Improvement: 0.00045
Mini-batch 319:
	After ITERATION: 15	Objective: 1067610.58	Old objective: 1067128.58	Improvement: 0.00045
Mini-batch 320:
	After ITERATION: 15	Objective: 1069347.33	Old objective: 1068856.14	Improvement: 0.00046
Mini-batch 321:
	After ITERATION: 15	Objective: 1094636.35	Old objective: 1094137.08	Improvement: 0.00046
Mini-batch 322:
	After ITERATION: 15	Objective: 1096606.83	Old objective: 1096113.41	Improvement: 0.00045
Mini-batch 323:
	After ITERATION: 15	Objective: 1038113.38	Old objective: 1037628.07	Improvement: 0.00047
Mini-batch 324:
	After ITERATION: 14	Objective: 1173746.46	Old objective: 1173168.28	Improvement: 0.00049
Mini-batch 325:
	After ITERATION: 15	Objective: 1104455.22	Old objective: 1103955.33	Improvement: 0.00045
Mini-batch 326:
	After ITERATION: 15	Objective: 1031683.65	Old objective: 1031186.31	Improvement: 0.00048
Mini-batch 327:
	After ITERATION: 15	Objective: 1081303.79	Old objective: 1080816.41	Improvement: 0.00045
Mini-batch 328:
	After ITERATION: 15	Objective: 1113955.30	Old objective: 1113462.86	Improvement: 0.00044
Mini-batch 329:
	After ITERATION: 15	Objective: 1113190.81	Old objective: 1112713.83	Improvement: 0.00043
Mini-batch 330:
	After ITERATION: 15	Objective: 1093214.67	Old objective: 1092716.61	Improvement: 0.00046
Mini-batch 331:
	After ITERATION: 15	Objective: 1037706.20	Old objective: 1037235.50	Improvement: 0.00045
Mini-batch 332:
	After ITERATION: 15	Objective: 1023487.36	Old objective: 1022989.60	Improvement: 0.00049
Mini-batch 333:
	After ITERATION: 15	Objective: 1013706.55	Old objective: 1013220.04	Improvement: 0.00048
Mini-batch 334:
	After ITERATION: 15	Objective: 1048025.35	Old objective: 1047560.67	Improvement: 0.00044
Mini-batch 335:
	After ITERATION: 15	Objective: 1026867.44	Old objective: 1026385.18	Improvement: 0.00047
Mini-batch 336:
	After ITERATION: 15	Objective: 1039473.65	Old objective: 1038973.57	Improvement: 0.00048
Mini-batch 337:
	After ITERATION: 15	Objective: 1073516.73	Old objective: 1073039.06	Improvement: 0.00045
Mini-batch 338:
	After ITERATION: 15	Objective: 1131881.75	Old objective: 1131397.01	Improvement: 0.00043
Mini-batch 339:
	After ITERATION: 15	Objective: 1081600.08	Old objective: 1081118.21	Improvement: 0.00045
Mini-batch 340:
	After ITERATION: 15	Objective: 1060541.36	Old objective: 1060060.66	Improvement: 0.00045
Mini-batch 341:
	After ITERATION: 15	Objective: 1085819.39	Old objective: 1085344.10	Improvement: 0.00044
Mini-batch 342:
	After ITERATION: 15	Objective: 1120657.77	Old objective: 1120163.03	Improvement: 0.00044
Mini-batch 343:
	After ITERATION: 15	Objective: 1086096.47	Old objective: 1085602.53	Improvement: 0.00045
Mini-batch 344:
	After ITERATION: 15	Objective: 1016054.40	Old objective: 1015585.74	Improvement: 0.00046
Mini-batch 345:
	After ITERATION: 15	Objective: 1024968.83	Old objective: 1024483.74	Improvement: 0.00047
Mini-batch 346:
	After ITERATION: 15	Objective: 1024766.33	Old objective: 1024270.37	Improvement: 0.00048
Mini-batch 347:
	After ITERATION: 15	Objective: 1071465.65	Old objective: 1070975.24	Improvement: 0.00046
Mini-batch 348:
	After ITERATION: 15	Objective: 1085319.86	Old objective: 1084823.87	Improvement: 0.00046
Mini-batch 349:
	After ITERATION: 15	Objective: 1069978.64	Old objective: 1069490.76	Improvement: 0.00046
Mini-batch 350:
	After ITERATION: 15	Objective: 1125202.56	Old objective: 1124713.97	Improvement: 0.00043
Mini-batch 351:
	After ITERATION: 15	Objective: 1109112.28	Old objective: 1108605.04	Improvement: 0.00046
Mini-batch 352:
	After ITERATION: 15	Objective: 1080803.82	Old objective: 1080348.42	Improvement: 0.00042
Mini-batch 353:
	After ITERATION: 15	Objective: 1112990.55	Old objective: 1112501.74	Improvement: 0.00044
Mini-batch 354:
	After ITERATION: 15	Objective: 1136617.48	Old objective: 1136130.21	Improvement: 0.00043
Mini-batch 355:
	After ITERATION: 15	Objective: 1130881.53	Old objective: 1130410.07	Improvement: 0.00042
Mini-batch 356:
	After ITERATION: 15	Objective: 1079733.41	Old objective: 1079255.95	Improvement: 0.00044
Mini-batch 357:
	After ITERATION: 15	Objective: 1079419.66	Old objective: 1078932.18	Improvement: 0.00045
Mini-batch 358:
	After ITERATION: 15	Objective: 1092988.90	Old objective: 1092499.83	Improvement: 0.00045
Mini-batch 359:
	After ITERATION: 15	Objective: 1088301.20	Old objective: 1087839.03	Improvement: 0.00042
Mini-batch 360:
	After ITERATION: 15	Objective: 1116411.89	Old objective: 1115935.17	Improvement: 0.00043
Mini-batch 361:
	After ITERATION: 15	Objective: 1120961.60	Old objective: 1120464.83	Improvement: 0.00044
Mini-batch 362:
	After ITERATION: 15	Objective: 1101373.62	Old objective: 1100895.62	Improvement: 0.00043
Mini-batch 363:
	After ITERATION: 15	Objective: 1052165.65	Old objective: 1051686.17	Improvement: 0.00046
Mini-batch 364:
	After ITERATION: 15	Objective: 1045082.61	Old objective: 1044591.19	Improvement: 0.00047
Mini-batch 365:
	After ITERATION: 14	Objective: 1106699.89	Old objective: 1106155.34	Improvement: 0.00049
Mini-batch 366:
	After ITERATION: 15	Objective: 1110061.80	Old objective: 1109568.83	Improvement: 0.00044
Mini-batch 367:
	After ITERATION: 15	Objective: 1049802.42	Old objective: 1049321.96	Improvement: 0.00046
Mini-batch 368:
	After ITERATION: 15	Objective: 1081693.68	Old objective: 1081217.30	Improvement: 0.00044
Mini-batch 369:
	After ITERATION: 15	Objective: 1074035.69	Old objective: 1073549.99	Improvement: 0.00045
Mini-batch 370:
	After ITERATION: 14	Objective: 1188333.55	Old objective: 1187756.43	Improvement: 0.00049
Mini-batch 371:
	After ITERATION: 14	Objective: 1190537.93	Old objective: 1189976.66	Improvement: 0.00047
Mini-batch 372:
	After ITERATION: 14	Objective: 265251.94	Old objective: 265135.87	Improvement: 0.00044

In [33]:
plot(online_coder_full.bound)
pass



In [34]:
ents = np.zeros((n_components, ))

for k in xrange(n_components):
    ents[k] = scipy.stats.entropy(online_coder_full.Eb[k])

In [35]:
idx = np.argsort(-ents)

In [36]:
plot(ents[idx], '-o')
pass



In [37]:
tagger = pmf.PoissonMF(n_components=n_components, random_state=98765, verbose=True)

In [38]:
tagger.set_components(online_coder_full.gamma_b[:, :K], online_coder_full.rho_b[:, :K])


Out[38]:
PoissonMF(max_iter=100, n_components=100, random_state=98765, smoothness=100,
     tol=0.0005, verbose=True)

In [39]:
Et = tagger.transform(X_test)


	After ITERATION: 15	Objective: 3303796.93	Old objective: 3302344.12	Improvement: 0.00044

In [40]:
Et /= Et.sum(axis=1, keepdims=True)
tags_predicted = Et.dot(online_coder_full.Eb[:, K:])
print tags_predicted.min(), tags_predicted.max()

div_factor = 3
tags_predicted = tags_predicted - div_factor * np.mean(tags_predicted, axis=0)


2.70206653139e-05 1.07069406765

In [41]:
predictat = 20
tags_true_binary = (y_test > 0)

print_out_metrics(tags_true_binary, tags_predicted, predictat)


Precision = 0.131 (0.008)
Recall = 0.154 (0.008)
F-score = 0.141
AROC = 0.718 (0.005)
AP = 0.122 (0.006)

In [ ]: