Evaluate Playlist generation / augmentation


In [1]:
%matplotlib inline
%load_ext autoreload
%autoreload 2

import os, sys, time, gzip
import pickle as pkl
import numpy as np
import pandas as pd
from sklearn.metrics import precision_recall_fscore_support, roc_auc_score, average_precision_score
from scipy.sparse import lil_matrix, issparse

import matplotlib.pyplot as plt
import seaborn as sns

In [2]:
sys.path.append('src')
from PCMLC import PCMLC as PClassificationMLC
from PCMLC import obj_pclassification
from BinaryRelevance import BinaryRelevance
from evaluate import f1_score_nowarn, calc_F1, calc_precisionK, evaluate_minibatch, evalPred

In [3]:
data_dir = 'data/aotm-2011/setting1'
fxtrain = os.path.join(data_dir, 'X_train.pkl.gz')
fytrain = os.path.join(data_dir, 'Y_train.pkl.gz')
fxdev   = os.path.join(data_dir, 'X_dev.pkl.gz')
fydev   = os.path.join(data_dir, 'Y_dev.pkl.gz')
fxtest  = os.path.join(data_dir, 'X_test.pkl.gz')
fytest  = os.path.join(data_dir, 'Y_test.pkl.gz')

In [26]:
TOPs = [5, 10, 20, 30, 50, 100, 200, 300, 500, 1000]

Data loading


In [5]:
X_train = pkl.load(gzip.open(fxtrain, 'rb'))
Y_train = pkl.load(gzip.open(fytrain, 'rb'))
X_dev   = pkl.load(gzip.open(fxdev,   'rb'))
Y_dev   = pkl.load(gzip.open(fydev,   'rb'))
X_test  = pkl.load(gzip.open(fxtest,  'rb'))
Y_test  = pkl.load(gzip.open(fytest,  'rb'))

In [6]:
print('Train: %15s %15s' % (X_train.shape, Y_train.shape))
print('Dev  : %15s %15s' % (X_dev.shape,   Y_dev.shape))
print('Test : %15s %15s' % (X_test.shape,  Y_test.shape))


Train:    (80100, 217)  (80100, 84710)
Dev  :    (11442, 217)  (11442, 84710)
Test :    (22886, 217)  (22886, 84710)

In [ ]:
def calc_auc_hitrate(estimator, X, Y, top=100, useLoop=False):
    N, D = X.shape
    K = Y.shape[1]
    assert Y.shape[0] == N
    assert top <= N
    if issparse(Y):
        #Y = Y.tocsc().astype(np.bool)
        npos = np.array(Y.sum(axis=0))[0]  # be careful with sparse matrix
    else:
        npos = np.sum(Y, axis=0)
    nzcol = np.nonzero(npos)[0]  # need columns with at least one True
    if useLoop is False:
        Y_pred = estimator.predict(X)
        auc = roc_auc_score(Y[:, nzcol], Y_pred[:, nzcol], average='macro')
        topix = np.argsort(-Y_pred, axis=0)[:top, :]
        hitrate = np.mean([np.sum(Y[topix[:, col], col]) / npos[col] for col in nzcol])
        return auc, hitrate
    else:
        W = estimator.W
        b = estimator.b
        assert W.shape == (K, D)
        aucs = []
        hitrates = []
        for col in nzcol:
            if (col+1) % 100 == 0:
                sys.stdout.write('\r{:,} / {:,}'.format(col+1, nzcol[-1]))
                sys.stdout.flush()
            pred = np.dot(X, W[col, :]) + b
            #aucs.append(roc_auc_score(Y[:, col], pred))  # TypeError: len() of unsized object
            y_true = Y[:, col].toarray()
            aucs.append(roc_auc_score(y_true, pred))
            topix = np.argsort(-pred)[:top]
            hitrates.append(np.sum(Y[topix, col]) / npos[col])
        return np.mean(aucs), np.mean(hitrates)

In [ ]:
def summary_auc(results):
    assert type(results) == list
    aucs = np.array([t[0] for t in results])
    nums = np.array([t[1] for t in results])
    return np.sum(aucs * nums) / np.sum(nums)

def calc_hitrate_pl(Y_true, Y_pred, top=100):
    assert Y_true.shape == Y_pred.shape
    assert top <= Y_true.shape[0]
    nzcol = np.nonzero(np.sum(Y_true, axis=0))[0]  # columns with at least one True
    topix = np.argsort(-Y_pred, axis=0)[:top, :]
    npos = np.sum(Y_true, axis=0)
    return np.array([np.sum(Y_true[topix[:, col], col]) / npos[col] for col in nzcol])

In [ ]:
def eval_pl(Y_true, Y_pred, top=100, useLoop=False):
    if useLoop is False:
        assert Y_true.shape == Y_pred.shape
        assert top <= Y_true.shape[0]
        nzcol = np.nonzero(np.sum(Y_true, axis=0))[0]  # columns with at least one True
        ncols = len(nzcol)
        topix = np.argsort(-Y_pred, axis=0)[:top, :]
        npos = np.sum(Y_true, axis=0)
        hr = np.mean([np.sum(Y_true[topix[:, j], j]) / npos[j] for j in nzcol])
        paks, valid_indices = calc_precisionK(Y_true.T, Y_pred.T)
        pak = np.mean(paks[valid_indices])
        auc = roc_auc_score(Y_true[:, nzcol], Y_pred[:, nzcol], average='macro')
        ap  = average_precision_score(Y_true[:, nzcol], Y_pred[:, nzcol], average='macro')
        nrr = mean_normalised_reciprocal_rank(Y_true, Y_pred)
    else:
        assert type(Y_true) == list
        assert type(Y_pred) == list
        assert len(Y_true) == len(Y_pred)
        hitrates, paks, aucs, aps, nrrs = [], [], [], [], []
        for j in range(len(Y_true)):
            if np.sum(Y_true[j]) < 1: continue   # filtering out cases where all ground truths are negative.
            gt = Y_true[j].reshape(-1)
            pred = Y_pred[j].reshape(-1)
            assert gt.shape == pred.shape
            assert top <= gt.shape[0]
            topix = np.argsort(-pred)[:top]
            hitrates.append(np.sum(gt[topix]) / np.sum(gt))
            #paks.append(calc_precisionK(gt.reshape(1,-1), pred.reshape(1,-1)))  # incorrect
            paks.append(evalPred(gt, pred, metricType='Precision@K'))
            aucs.append(roc_auc_score(gt, pred))
            aps.append(average_precision_score(gt, pred))
            nrrs.append(mean_normalised_reciprocal_rank(gt.reshape(-1,1), pred.reshape(-1,1)))
        hr, pak, auc, ap, nrr = [np.mean(x) for x in [hitrates, paks, aucs, aps, nrrs]]
        ncols = len(paks)
      
    print('Average over %d columns' % ncols)
    print('%-20s %.4f' % ('Mean HitRate@100:', hr))
    print('%-20s %.4f' % ('Mean P@K:', pak))
    print('%-20s %.4f' % ('Mean AUC:', auc))
    print('%-20s %.4f' % ('MAP:', ap))
    print('%-20s %.4f' % ('Mean NRR:', nrr))

In [ ]:
def calc_auc_pl(Y_true, Y_pred, useLoop=False):
    if useLoop is False:
        assert Y_true.shape == Y_pred.shape
        npos = np.asarray(np.sum(Y_true, axis=0)).reshape(-1)  # 1D array, works for both sparse and dense matrix
        nzcol = np.nonzero(npos)[0]  # columns with at least one True
        Y_part = Y_true[:, nzcol]
        if issparse(Y_part):
            Y_part = Y_part.toarray()
        auc = roc_auc_score(Y_part, Y_pred[:, nzcol], average='macro')
        return (auc, len(nzcol))
    else:
        assert type(Y_true) == list
        assert type(Y_pred) == list
        assert len(Y_true) == len(Y_pred)
        aucs = []
        for j in range(len(Y_true)):
            if np.sum(Y_true[j]) < 1: continue   # filtering out cases where all ground truths are negative.
            gt = Y_true[j].reshape(-1)
            pred = Y_pred[j].reshape(-1)
            assert gt.shape == pred.shape
            aucs.append(roc_auc_score(gt, pred))
        return (np.mean(aucs), len(aucs))

In [ ]:
def calc_hitrate_pl(Y_true, Y_pred, tops=[100], useLoop=False):
    """
        Compute hitrate at top-N.
    """
    if useLoop is False:
        assert Y_true.shape == Y_pred.shape
        assert type(tops) == list
        hitrates = []
        sortix = np.argsort(-Y_pred, axis=0)
        npos = np.asarray(np.sum(Y_true, axis=0)).reshape(-1)  # 1D array, works for both sparse and dense matrix
        nzcol = np.nonzero(npos)[0]  # columns with at least one True
        for top in tops:
            assert 0 < top <= Y_true.shape[0]
            topix = sortix[:top, :]
            hr = np.mean([np.sum(Y_true[topix[:, col], col]) / npos[col] for col in nzcol])
            hitrates.append(hr)
        return (hitrates, len(nzcol))
    else:
        assert type(Y_true) == list
        assert type(Y_pred) == list
        assert len(Y_true) == len(Y_pred)
        assert type(tops) == list
        top_dict = {top: [] for top in tops}
        for j in range(len(Y_true)):
            if np.sum(Y_true[j]) < 1: continue   # filtering out cases where all ground truths are negative.
            gt = Y_true[j].reshape(-1)
            pred = Y_pred[j].reshape(-1)
            sortix = np.argsort(-pred)
            assert gt.shape == pred.shape
            for top in tops:
                assert 0 < top <= gt.shape[0]
                topix = sortix[:top]
                top_dict[top].append(np.sum(gt[topix]) / np.sum(gt))
        ncols = len(top_dict[tops[0]])
        for top in tops:
            assert ncols == len(top_dict[top])
        return ([np.mean(top_dict[top]) for top in tops], ncols)

MLR on dev set


In [7]:
C1 = [0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30, 100, 300, 1000, 3000]
C2 = [0.01, 0.1, 0.2, 0.4, 0.6, 0.8, 1, 3, 10, 30, 100, 300, 1000]
C3 = [0.01, 0.1, 0.3, 1, 2, 4, 6, 8, 10, 20, 30, 100, 300, 1000]
P = [1, 2, 3, 4, 5, 6]

In [ ]:
%%script false
for c1 in C1:
    for c2 in C2:
        for c3 in C3:
            for p in P:
                fname = os.path.join(data_dir, 'mlr-aotm2011-C-%g-%g-%g-p-%g.pkl' % (c1, c2, c3, p))
                if os.path.exists(fname):
                    clf = pkl.load(open(fname, 'rb'))
                    print(clf)
                    auc_dev, hr_dev = calc_auc_hitrate(clf, X_dev, Y_dev, useLoop=True)
                    print('{:10s}: AUC = {:.4f}, HitRate@100 = {:.4f}'.format('Dev set', auc_dev, hr_dev))
                    auc_test, hr_test = calc_auc_hitrate(clf, X_test, Y_test, useLoop=True)
                    print('{:10s}: AUC = {:.4f}, HitRate@100 = {:.4f}'.format('Test set', auc_test, hr_test))
                    print('-'*50, '\n')

In [8]:
def calc_auc_w(W, b, X, Y):
    aucs = []
    for j in range(Y.shape[1]):
        if (j+1) % 100 == 0:
            sys.stdout.write('\r%d / %d' % (j+1, Y.shape[1]))
            sys.stdout.flush()
        y_true = Y[:, j]
        if issparse(Y):
            y_true = y_true.toarray().reshape(-1)
        if np.sum(y_true) < 1: continue
        y_pred = np.dot(X, W[j, :]) + b
        aucs.append(roc_auc_score(y_true, y_pred))
    return aucs

In [ ]:
def calc_hitrate_w(W, b, X, Y, tops=[100]):
    """
        Compute hitrate at top-N.
    """
    top_dict = {top: [] for top in tops}
    for j in range(Y.shape[1]):
        if (j+1) % 100 == 0:
            sys.stdout.write('\r%d / %d' % (j+1, Y.shape[1]))
            sys.stdout.flush()
        y_true = Y[:, j]
        if issparse(Y):
            y_true = y_true.toarray().reshape(-1)
        npos = np.sum(y_true)
        if npos < 1: continue
        y_pred = np.dot(X, W[j, :]) + b
        y_pred = y_pred.reshape(-1)
        sortix = np.argsort(-y_pred)
        assert y_true.shape == y_pred.shape
        for top in tops:
            assert 0 < top <= y_true.shape[0]
            topix = sortix[:top]
            top_dict[top].append(np.sum(y_true[topix]) / npos)
    ncols = len(top_dict[tops[0]])
    for top in tops:
        assert ncols == len(top_dict[top])
    return ({top: np.mean(top_dict[top]) for top in tops}, ncols)

In [28]:
w = np.load('data/aotm-2011/setting1/mlr_lineareg/both-1000-0.01-30-5-latest.npy')
aucs1 = calc_auc_w(W=w[1:].reshape(K, D), b=w[0], X=X_dev, Y=Y_dev)
print('\n%.4f, %d / %d' % (np.mean(aucs1), len(aucs1), Y_dev.shape[1]))


84700 / 84710
0.6662, 37886 / 84710

In [29]:
w = np.load('data/aotm-2011/setting1/mlr_lineareg/both-1000-0.01-30-5-latest.npy')
aucs2 = calc_auc_w(W=w[1:].reshape(K, D), b=w[0], X=X_test, Y=Y_test)
print('\n%.4f, %d / %d' % (np.mean(aucs2), len(aucs2), Y_test.shape[1]))


84700 / 84710
0.6724, 52797 / 84710

In [32]:
for c1 in C1:
    if c1 == 0.03: continue
    for p in P:
        fname = os.path.join(data_dir, 'mlr_samples/mlr-aotm2011-samples-C-%g-1-1-p-%d.pkl' % (c1, p))
        if not os.path.exists(fname):continue
        print(fname)
        mlr = pkl.load(open(fname, 'rb'))
        aucs_mlr = calc_auc_w(W=mlr.W, b=mlr.b, X=X_test, Y=Y_test)
        print('\n%.5f, %d / %d' % (np.mean(aucs_mlr), len(aucs_mlr), Y_test.shape[1]))


data/aotm-2011/setting1/mlr_samples/mlr-aotm2011-samples-C-0.1-1-1-p-1.pkl
84700 / 84710
0.6031, 52797 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-aotm2011-samples-C-0.1-1-1-p-2.pkl
84700 / 84710
0.6099, 52797 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-aotm2011-samples-C-0.1-1-1-p-3.pkl
84700 / 84710
0.6161, 52797 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-aotm2011-samples-C-0.1-1-1-p-4.pkl
84700 / 84710
0.6102, 52797 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-aotm2011-samples-C-0.1-1-1-p-5.pkl
84700 / 84710
0.6133, 52797 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-aotm2011-samples-C-0.3-1-1-p-1.pkl
84700 / 84710
0.6423, 52797 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-aotm2011-samples-C-0.3-1-1-p-2.pkl
84700 / 84710
0.6414, 52797 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-aotm2011-samples-C-0.3-1-1-p-3.pkl
84700 / 84710
0.6441, 52797 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-aotm2011-samples-C-0.3-1-1-p-4.pkl
84700 / 84710
0.6422, 52797 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-aotm2011-samples-C-0.3-1-1-p-5.pkl
84700 / 84710
0.6413, 52797 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-aotm2011-samples-C-1-1-1-p-1.pkl
84700 / 84710
0.6564, 52797 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-aotm2011-samples-C-1-1-1-p-2.pkl
84700 / 84710
0.6572, 52797 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-aotm2011-samples-C-1-1-1-p-3.pkl
84700 / 84710
0.6571, 52797 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-aotm2011-samples-C-3-1-1-p-2.pkl
84700 / 84710
0.6579, 52797 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-aotm2011-samples-C-3-1-1-p-3.pkl
84700 / 84710
0.6583, 52797 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-aotm2011-samples-C-3-1-1-p-4.pkl
84700 / 84710
0.6576, 52797 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-aotm2011-samples-C-30-1-1-p-3.pkl
84700 / 84710
0.6579, 52797 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-aotm2011-samples-C-30-1-1-p-4.pkl
84700 / 84710
0.6585, 52797 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-aotm2011-samples-C-100-1-1-p-1.pkl
84700 / 84710
0.6585, 52797 / 84710

Loss: sample weighting


In [11]:
D = X_train.shape[1]
K = Y_train.shape[1]
for c1 in C1:
    for p in P:
        fname = os.path.join(data_dir, 'mlr_samples/mlr-N-samples-%g-1-1-%d-latest.npy' % (c1, p))
        if not os.path.exists(fname):continue
        print(fname)
        w = np.load(fname)
        aucs = calc_auc_w(W=w[1:].reshape(K, D), b=w[0], X=X_dev, Y=Y_dev)
        print('\n%.5f, %d / %d\n' % (np.mean(aucs), len(aucs), Y_dev.shape[1]))


data/aotm-2011/setting1/mlr_samples/mlr-N-samples-0.01-1-1-1-latest.npy
84700 / 84710
0.5104, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-0.01-1-1-2-latest.npy
84700 / 84710
0.4827, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-0.01-1-1-3-latest.npy
84700 / 84710
0.4734, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-0.01-1-1-4-latest.npy
84700 / 84710
0.4866, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-0.01-1-1-5-latest.npy
84700 / 84710
0.4948, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-0.01-1-1-6-latest.npy
84700 / 84710
0.4815, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-0.03-1-1-1-latest.npy
84700 / 84710
0.5480, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-0.03-1-1-2-latest.npy
84700 / 84710
0.5836, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-0.03-1-1-3-latest.npy
84700 / 84710
0.5859, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-0.03-1-1-4-latest.npy
84700 / 84710
0.5838, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-0.03-1-1-5-latest.npy
84700 / 84710
0.5915, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-0.03-1-1-6-latest.npy
84700 / 84710
0.5951, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-0.1-1-1-1-latest.npy
84700 / 84710
0.6066, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-0.1-1-1-2-latest.npy
84700 / 84710
0.5992, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-0.1-1-1-3-latest.npy
84700 / 84710
0.6021, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-0.1-1-1-4-latest.npy
84700 / 84710
0.5980, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-0.1-1-1-5-latest.npy
84700 / 84710
0.5999, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-0.1-1-1-6-latest.npy
84700 / 84710
0.6048, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-0.3-1-1-1-latest.npy
84700 / 84710
0.6374, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-0.3-1-1-2-latest.npy
84700 / 84710
0.6357, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-0.3-1-1-3-latest.npy
84700 / 84710
0.6355, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-0.3-1-1-4-latest.npy
84700 / 84710
0.6371, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-0.3-1-1-5-latest.npy
84700 / 84710
0.6361, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-0.3-1-1-6-latest.npy
84700 / 84710
0.6342, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-1-1-1-1-latest.npy
84700 / 84710
0.6493, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-1-1-1-2-latest.npy
84700 / 84710
0.6499, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-1-1-1-3-latest.npy
84700 / 84710
0.6488, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-1-1-1-4-latest.npy
84700 / 84710
0.6490, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-1-1-1-5-latest.npy
84700 / 84710
0.6491, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-1-1-1-6-latest.npy
84700 / 84710
0.6488, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-3-1-1-1-latest.npy
84700 / 84710
0.6506, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-3-1-1-2-latest.npy
84700 / 84710
0.6503, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-3-1-1-3-latest.npy
84700 / 84710
0.6495, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-3-1-1-4-latest.npy
84700 / 84710
0.6504, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-3-1-1-5-latest.npy
84700 / 84710
0.6496, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-3-1-1-6-latest.npy
84700 / 84710
0.6504, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-10-1-1-1-latest.npy
84700 / 84710
0.6504, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-10-1-1-2-latest.npy
84700 / 84710
0.6507, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-10-1-1-3-latest.npy
84700 / 84710
0.6510, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-10-1-1-4-latest.npy
84700 / 84710
0.6507, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-10-1-1-5-latest.npy
84700 / 84710
0.6506, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-10-1-1-6-latest.npy
84700 / 84710
0.6506, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-30-1-1-1-latest.npy
84700 / 84710
0.6510, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-30-1-1-2-latest.npy
84700 / 84710
0.6506, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-30-1-1-3-latest.npy
84700 / 84710
0.6508, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-30-1-1-4-latest.npy
84700 / 84710
0.6506, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-30-1-1-5-latest.npy
84700 / 84710
0.6502, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-30-1-1-6-latest.npy
84700 / 84710
0.6502, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-100-1-1-1-latest.npy
84700 / 84710
0.6506, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-100-1-1-2-latest.npy
84700 / 84710
0.6503, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-100-1-1-3-latest.npy
84700 / 84710
0.6509, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-100-1-1-4-latest.npy
84700 / 84710
0.6510, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-100-1-1-5-latest.npy
84700 / 84710
0.6499, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-100-1-1-6-latest.npy
84700 / 84710
0.6511, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-300-1-1-1-latest.npy
84700 / 84710
0.6503, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-300-1-1-2-latest.npy
84700 / 84710
0.6500, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-300-1-1-3-latest.npy
84700 / 84710
0.6502, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-300-1-1-4-latest.npy
84700 / 84710
0.6505, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-300-1-1-5-latest.npy
84700 / 84710
0.6512, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-300-1-1-6-latest.npy
84700 / 84710
0.6507, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-1000-1-1-1-latest.npy
84700 / 84710
0.6505, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-1000-1-1-2-latest.npy
84700 / 84710
0.6507, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-1000-1-1-3-latest.npy
84700 / 84710
0.6507, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-1000-1-1-4-latest.npy
84700 / 84710
0.6506, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-1000-1-1-5-latest.npy
84700 / 84710
0.6510, 37886 / 84710
data/aotm-2011/setting1/mlr_samples/mlr-N-samples-1000-1-1-6-latest.npy
84700 / 84710
0.6502, 37886 / 84710

In [13]:
print('Choose: mlr_samples/mlr-N-samples-10-1-1-3-latest.npy')


Choose: mlr_samples/mlr-N-samples-30-1-1-1-latest.npy

In [13]:
print('Choose: mlr_samples/mlr-N-samples-30-1-1-1-latest.npy')


Choose: mlr_samples/mlr-N-samples-30-1-1-1-latest.npy

Loss: labels weighting


In [14]:
D = X_train.shape[1]
K = Y_train.shape[1]
for c1 in C1:
    for p in P:
        fname = os.path.join(data_dir, 'mlr_labels/mlr-N-labels-%g-1-1-%d-latest.npy' % (c1, p))
        if not os.path.exists(fname):continue
        print(fname)
        w = np.load(fname)
        aucs = calc_auc_w(W=w[1:].reshape(K, D), b=w[0], X=X_dev, Y=Y_dev)
        print('\n%.5f, %d / %d\n' % (np.mean(aucs), len(aucs), Y_dev.shape[1]))


data/aotm-2011/setting1/mlr_labels/mlr-N-labels-0.01-1-1-1-latest.npy
84700 / 84710
0.50917, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-0.01-1-1-2-latest.npy
84700 / 84710
0.52261, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-0.01-1-1-3-latest.npy
84700 / 84710
0.51337, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-0.01-1-1-4-latest.npy
84700 / 84710
0.47382, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-0.01-1-1-5-latest.npy
84700 / 84710
0.51257, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-0.01-1-1-6-latest.npy
84700 / 84710
0.51712, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-0.03-1-1-1-latest.npy
84700 / 84710
0.54490, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-0.03-1-1-2-latest.npy
84700 / 84710
0.59225, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-0.03-1-1-3-latest.npy
84700 / 84710
0.59218, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-0.03-1-1-4-latest.npy
84700 / 84710
0.59361, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-0.03-1-1-5-latest.npy
84700 / 84710
0.59466, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-0.03-1-1-6-latest.npy
84700 / 84710
0.59130, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-0.1-1-1-1-latest.npy
84700 / 84710
0.60103, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-0.1-1-1-2-latest.npy
84700 / 84710
0.60204, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-0.1-1-1-3-latest.npy
84700 / 84710
0.59542, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-0.1-1-1-4-latest.npy
84700 / 84710
0.60101, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-0.1-1-1-5-latest.npy
84700 / 84710
0.59957, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-0.1-1-1-6-latest.npy
84700 / 84710
0.59970, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-0.3-1-1-1-latest.npy
84700 / 84710
0.62892, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-0.3-1-1-2-latest.npy
84700 / 84710
0.62884, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-0.3-1-1-3-latest.npy
84700 / 84710
0.62898, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-0.3-1-1-4-latest.npy
84700 / 84710
0.62875, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-0.3-1-1-5-latest.npy
84700 / 84710
0.62980, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-0.3-1-1-6-latest.npy
84700 / 84710
0.62477, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-1-1-1-1-latest.npy
84700 / 84710
0.63456, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-1-1-1-2-latest.npy
84700 / 84710
0.63457, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-1-1-1-3-latest.npy
84700 / 84710
0.63490, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-1-1-1-4-latest.npy
84700 / 84710
0.63483, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-1-1-1-5-latest.npy
84700 / 84710
0.63389, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-1-1-1-6-latest.npy
84700 / 84710
0.63432, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-3-1-1-1-latest.npy
84700 / 84710
0.63488, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-3-1-1-2-latest.npy
84700 / 84710
0.63441, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-3-1-1-3-latest.npy
84700 / 84710
0.63500, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-3-1-1-4-latest.npy
84700 / 84710
0.63469, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-3-1-1-5-latest.npy
84700 / 84710
0.63513, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-3-1-1-6-latest.npy
84700 / 84710
0.63440, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-10-1-1-1-latest.npy
84700 / 84710
0.63488, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-10-1-1-2-latest.npy
84700 / 84710
0.63451, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-10-1-1-3-latest.npy
84700 / 84710
0.63434, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-10-1-1-4-latest.npy
84700 / 84710
0.63535, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-10-1-1-5-latest.npy
84700 / 84710
0.63497, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-10-1-1-6-latest.npy
84700 / 84710
0.63469, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-30-1-1-1-latest.npy
84700 / 84710
0.63404, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-30-1-1-2-latest.npy
84700 / 84710
0.63504, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-30-1-1-3-latest.npy
84700 / 84710
0.63553, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-30-1-1-4-latest.npy
84700 / 84710
0.63519, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-30-1-1-5-latest.npy
84700 / 84710
0.63490, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-30-1-1-6-latest.npy
84700 / 84710
0.63469, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-100-1-1-1-latest.npy
84700 / 84710
0.63485, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-100-1-1-2-latest.npy
84700 / 84710
0.63430, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-100-1-1-3-latest.npy
84700 / 84710
0.63434, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-100-1-1-4-latest.npy
84700 / 84710
0.63495, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-100-1-1-5-latest.npy
84700 / 84710
0.63460, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-100-1-1-6-latest.npy
84700 / 84710
0.63497, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-300-1-1-1-latest.npy
84700 / 84710
0.63504, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-300-1-1-2-latest.npy
84700 / 84710
0.63535, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-300-1-1-3-latest.npy
84700 / 84710
0.63482, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-300-1-1-4-latest.npy
84700 / 84710
0.63515, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-300-1-1-5-latest.npy
84700 / 84710
0.63479, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-300-1-1-6-latest.npy
84700 / 84710
0.63400, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-1000-1-1-1-latest.npy
84700 / 84710
0.63438, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-1000-1-1-2-latest.npy
84700 / 84710
0.63542, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-1000-1-1-3-latest.npy
84700 / 84710
0.63420, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-1000-1-1-4-latest.npy
84700 / 84710
0.63459, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-1000-1-1-5-latest.npy
84700 / 84710
0.63465, 37886 / 84710

data/aotm-2011/setting1/mlr_labels/mlr-N-labels-1000-1-1-6-latest.npy
84700 / 84710
0.63513, 37886 / 84710


In [15]:
print('Choose:', 'data/aotm-2011/setting1/mlr_labels/mlr-N-labels-30-1-1-3-latest.npy')


Choose: data/aotm-2011/setting1/mlr_labels/mlr-N-labels-30-1-1-3-latest.npy

Loss: both samples and labels weighting


In [16]:
D = X_train.shape[1]
K = Y_train.shape[1]
for c1 in C1:
    for c2 in C2:
        for p in P:
            fname = os.path.join(data_dir, 'mlr_both/mlr-N-both-%g-%g-1-%d-latest.npy' % (c1, c2, p))
            if not os.path.exists(fname):continue
            print(fname)
            w = np.load(fname)
            aucs = calc_auc_w(W=w[1:].reshape(K, D), b=w[0], X=X_dev, Y=Y_dev)
            print('\n%.5f, %d / %d\n' % (np.mean(aucs), len(aucs), Y_dev.shape[1]))


data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-0.01-1-1-latest.npy
84700 / 84710
0.52103, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-0.01-1-2-latest.npy
84700 / 84710
0.50021, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-0.01-1-3-latest.npy
84700 / 84710
0.51369, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-0.01-1-4-latest.npy
84700 / 84710
0.49892, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-0.01-1-5-latest.npy
84700 / 84710
0.50855, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-0.01-1-6-latest.npy
84700 / 84710
0.48563, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-0.1-1-1-latest.npy
84700 / 84710
0.50689, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-0.1-1-2-latest.npy
84700 / 84710
0.50539, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-0.1-1-3-latest.npy
84700 / 84710
0.49757, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-0.1-1-4-latest.npy
84700 / 84710
0.50536, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-0.1-1-5-latest.npy
84700 / 84710
0.50330, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-0.1-1-6-latest.npy
84700 / 84710
0.47963, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-1-1-1-latest.npy
84700 / 84710
0.50017, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-1-1-2-latest.npy
84700 / 84710
0.50946, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-1-1-3-latest.npy
84700 / 84710
0.48924, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-1-1-4-latest.npy
84700 / 84710
0.51811, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-1-1-5-latest.npy
84700 / 84710
0.51596, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-1-1-6-latest.npy
84700 / 84710
0.51450, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-3-1-1-latest.npy
84700 / 84710
0.50427, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-3-1-2-latest.npy
84700 / 84710
0.50789, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-3-1-3-latest.npy
84700 / 84710
0.50085, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-3-1-4-latest.npy
84700 / 84710
0.51577, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-3-1-5-latest.npy
84700 / 84710
0.52213, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-3-1-6-latest.npy
84700 / 84710
0.48419, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-10-1-1-latest.npy
84700 / 84710
0.49141, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-10-1-2-latest.npy
84700 / 84710
0.50031, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-10-1-3-latest.npy
84700 / 84710
0.50251, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-10-1-4-latest.npy
84700 / 84710
0.49760, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-10-1-5-latest.npy
84700 / 84710
0.48545, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-10-1-6-latest.npy
84700 / 84710
0.48492, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-30-1-1-latest.npy
84700 / 84710
0.49210, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-30-1-2-latest.npy
84700 / 84710
0.53281, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-30-1-3-latest.npy
84700 / 84710
0.49777, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-30-1-4-latest.npy
84700 / 84710
0.50029, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-30-1-5-latest.npy
84700 / 84710
0.51589, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-30-1-6-latest.npy
84700 / 84710
0.50859, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-100-1-1-latest.npy
84700 / 84710
0.49868, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-100-1-2-latest.npy
84700 / 84710
0.51456, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-100-1-3-latest.npy
84700 / 84710
0.50652, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-100-1-4-latest.npy
84700 / 84710
0.50552, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-100-1-5-latest.npy
84700 / 84710
0.50147, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-100-1-6-latest.npy
84700 / 84710
0.50481, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-300-1-1-latest.npy
84700 / 84710
0.51045, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-300-1-2-latest.npy
84700 / 84710
0.50785, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-300-1-3-latest.npy
84700 / 84710
0.51363, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-300-1-4-latest.npy
84700 / 84710
0.49921, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-300-1-5-latest.npy
84700 / 84710
0.50600, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-300-1-6-latest.npy
84700 / 84710
0.51235, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-1000-1-1-latest.npy
84700 / 84710
0.50434, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-1000-1-2-latest.npy
84700 / 84710
0.51806, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-1000-1-3-latest.npy
84700 / 84710
0.50015, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-1000-1-4-latest.npy
84700 / 84710
0.49211, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-1000-1-5-latest.npy
84700 / 84710
0.51209, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.01-1000-1-6-latest.npy
84700 / 84710
0.49924, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-0.01-1-1-latest.npy
84700 / 84710
0.54454, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-0.01-1-2-latest.npy
84700 / 84710
0.58104, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-0.01-1-3-latest.npy
84700 / 84710
0.59485, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-0.01-1-4-latest.npy
84700 / 84710
0.59385, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-0.01-1-5-latest.npy
84700 / 84710
0.59255, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-0.01-1-6-latest.npy
84700 / 84710
0.59411, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-0.1-1-1-latest.npy
84700 / 84710
0.53591, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-0.1-1-2-latest.npy
84700 / 84710
0.58582, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-0.1-1-3-latest.npy
84700 / 84710
0.59370, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-0.1-1-4-latest.npy
84700 / 84710
0.59420, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-0.1-1-5-latest.npy
84700 / 84710
0.59214, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-0.1-1-6-latest.npy
84700 / 84710
0.59348, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-1-1-1-latest.npy
84700 / 84710
0.54172, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-1-1-2-latest.npy
84700 / 84710
0.58175, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-1-1-3-latest.npy
84700 / 84710
0.59462, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-1-1-4-latest.npy
84700 / 84710
0.59550, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-1-1-5-latest.npy
84700 / 84710
0.59368, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-1-1-6-latest.npy
84700 / 84710
0.59211, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-3-1-1-latest.npy
84700 / 84710
0.54633, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-3-1-2-latest.npy
84700 / 84710
0.58373, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-3-1-3-latest.npy
84700 / 84710
0.59462, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-3-1-4-latest.npy
84700 / 84710
0.59256, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-3-1-5-latest.npy
84700 / 84710
0.59411, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-3-1-6-latest.npy
84700 / 84710
0.59506, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-10-1-1-latest.npy
84700 / 84710
0.53824, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-10-1-2-latest.npy
84700 / 84710
0.58482, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-10-1-3-latest.npy
84700 / 84710
0.59600, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-10-1-4-latest.npy
84700 / 84710
0.59451, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-10-1-5-latest.npy
84700 / 84710
0.59376, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-10-1-6-latest.npy
84700 / 84710
0.59297, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-30-1-1-latest.npy
84700 / 84710
0.55196, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-30-1-2-latest.npy
84700 / 84710
0.58008, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-30-1-3-latest.npy
84700 / 84710
0.59308, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-30-1-4-latest.npy
84700 / 84710
0.59524, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-30-1-5-latest.npy
84700 / 84710
0.59213, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-30-1-6-latest.npy
84700 / 84710
0.59175, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-100-1-1-latest.npy
84700 / 84710
0.55424, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-100-1-2-latest.npy
84700 / 84710
0.59353, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-100-1-3-latest.npy
84700 / 84710
0.59392, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-100-1-4-latest.npy
84700 / 84710
0.59377, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-100-1-5-latest.npy
84700 / 84710
0.59177, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-100-1-6-latest.npy
84700 / 84710
0.59410, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-300-1-1-latest.npy
84700 / 84710
0.56275, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-300-1-2-latest.npy
84700 / 84710
0.59339, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-300-1-3-latest.npy
84700 / 84710
0.59405, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-300-1-4-latest.npy
84700 / 84710
0.59302, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-300-1-5-latest.npy
84700 / 84710
0.59369, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-300-1-6-latest.npy
84700 / 84710
0.59394, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-1000-1-1-latest.npy
84700 / 84710
0.59131, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-1000-1-2-latest.npy
84700 / 84710
0.59413, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-1000-1-3-latest.npy
84700 / 84710
0.59340, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-1000-1-4-latest.npy
84700 / 84710
0.59384, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-1000-1-5-latest.npy
84700 / 84710
0.59467, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.03-1000-1-6-latest.npy
84700 / 84710
0.59500, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-0.01-1-1-latest.npy
84700 / 84710
0.59894, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-0.01-1-2-latest.npy
84700 / 84710
0.60350, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-0.01-1-3-latest.npy
84700 / 84710
0.59755, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-0.01-1-4-latest.npy
84700 / 84710
0.60746, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-0.01-1-5-latest.npy
84700 / 84710
0.60537, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-0.01-1-6-latest.npy
84700 / 84710
0.59915, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-0.1-1-1-latest.npy
84700 / 84710
0.60059, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-0.1-1-2-latest.npy
84700 / 84710
0.60804, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-0.1-1-3-latest.npy
84700 / 84710
0.60291, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-0.1-1-4-latest.npy
84700 / 84710
0.60110, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-0.1-1-5-latest.npy
84700 / 84710
0.60444, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-0.1-1-6-latest.npy
84700 / 84710
0.60004, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-1-1-1-latest.npy
84700 / 84710
0.59967, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-1-1-2-latest.npy
84700 / 84710
0.60495, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-1-1-3-latest.npy
84700 / 84710
0.60163, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-1-1-4-latest.npy
84700 / 84710
0.60769, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-1-1-5-latest.npy
84700 / 84710
0.60259, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-1-1-6-latest.npy
84700 / 84710
0.59684, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-3-1-1-latest.npy
84700 / 84710
0.59961, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-3-1-2-latest.npy
84700 / 84710
0.60108, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-3-1-3-latest.npy
84700 / 84710
0.60158, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-3-1-4-latest.npy
84700 / 84710
0.59740, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-3-1-5-latest.npy
84700 / 84710
0.60715, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-3-1-6-latest.npy
84700 / 84710
0.59989, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-10-1-1-latest.npy
84700 / 84710
0.59834, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-10-1-2-latest.npy
84700 / 84710
0.60904, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-10-1-3-latest.npy
84700 / 84710
0.59924, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-10-1-4-latest.npy
84700 / 84710
0.58798, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-10-1-5-latest.npy
84700 / 84710
0.59830, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-10-1-6-latest.npy
84700 / 84710
0.59698, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-30-1-1-latest.npy
84700 / 84710
0.60097, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-30-1-2-latest.npy
84700 / 84710
0.60331, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-30-1-3-latest.npy
84700 / 84710
0.59977, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-30-1-4-latest.npy
84700 / 84710
0.59602, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-30-1-5-latest.npy
84700 / 84710
0.60047, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-30-1-6-latest.npy
84700 / 84710
0.59761, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-100-1-1-latest.npy
84700 / 84710
0.59186, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-100-1-2-latest.npy
84700 / 84710
0.60475, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-100-1-3-latest.npy
84700 / 84710
0.59696, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-100-1-4-latest.npy
84700 / 84710
0.59902, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-100-1-5-latest.npy
84700 / 84710
0.60484, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-100-1-6-latest.npy
84700 / 84710
0.59516, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-300-1-1-latest.npy
84700 / 84710
0.60419, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-300-1-2-latest.npy
84700 / 84710
0.59446, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-300-1-3-latest.npy
84700 / 84710
0.58844, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-300-1-4-latest.npy
84700 / 84710
0.59965, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-300-1-5-latest.npy
84700 / 84710
0.59972, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-300-1-6-latest.npy
84700 / 84710
0.60310, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-1000-1-1-latest.npy
84700 / 84710
0.60035, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-1000-1-2-latest.npy
84700 / 84710
0.59815, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-1000-1-3-latest.npy
84700 / 84710
0.59796, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-1000-1-4-latest.npy
84700 / 84710
0.60382, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-1000-1-5-latest.npy
84700 / 84710
0.59488, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.1-1000-1-6-latest.npy
84700 / 84710
0.60565, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-0.01-1-1-latest.npy
84700 / 84710
0.63645, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-0.01-1-2-latest.npy
84700 / 84710
0.63662, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-0.01-1-3-latest.npy
84700 / 84710
0.63371, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-0.01-1-4-latest.npy
84700 / 84710
0.63660, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-0.01-1-5-latest.npy
84700 / 84710
0.63558, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-0.01-1-6-latest.npy
84700 / 84710
0.63760, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-0.1-1-1-latest.npy
84700 / 84710
0.63560, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-0.1-1-2-latest.npy
84700 / 84710
0.63421, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-0.1-1-3-latest.npy
84700 / 84710
0.63682, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-0.1-1-4-latest.npy
84700 / 84710
0.63499, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-0.1-1-5-latest.npy
84700 / 84710
0.63527, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-0.1-1-6-latest.npy
84700 / 84710
0.63487, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-1-1-1-latest.npy
84700 / 84710
0.63196, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-1-1-2-latest.npy
84700 / 84710
0.63134, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-1-1-3-latest.npy
84700 / 84710
0.63102, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-1-1-4-latest.npy
84700 / 84710
0.63180, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-1-1-5-latest.npy
84700 / 84710
0.63116, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-1-1-6-latest.npy
84700 / 84710
0.63142, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-3-1-1-latest.npy
84700 / 84710
0.63175, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-3-1-2-latest.npy
84700 / 84710
0.63248, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-3-1-3-latest.npy
84700 / 84710
0.62938, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-3-1-4-latest.npy
84700 / 84710
0.62809, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-3-1-5-latest.npy
84700 / 84710
0.62854, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-3-1-6-latest.npy
84700 / 84710
0.62936, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-10-1-1-latest.npy
84700 / 84710
0.62979, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-10-1-2-latest.npy
84700 / 84710
0.63203, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-10-1-3-latest.npy
84700 / 84710
0.62770, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-10-1-4-latest.npy
84700 / 84710
0.62916, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-10-1-5-latest.npy
84700 / 84710
0.62658, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-10-1-6-latest.npy
84700 / 84710
0.63034, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-30-1-1-latest.npy
84700 / 84710
0.63019, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-30-1-2-latest.npy
84700 / 84710
0.62871, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-30-1-3-latest.npy
84700 / 84710
0.63056, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-30-1-4-latest.npy
84700 / 84710
0.63026, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-30-1-5-latest.npy
84700 / 84710
0.63012, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-30-1-6-latest.npy
84700 / 84710
0.62858, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-100-1-1-latest.npy
84700 / 84710
0.62522, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-100-1-2-latest.npy
84700 / 84710
0.62954, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-100-1-3-latest.npy
84700 / 84710
0.62731, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-100-1-4-latest.npy
84700 / 84710
0.62998, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-100-1-5-latest.npy
84700 / 84710
0.62861, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-100-1-6-latest.npy
84700 / 84710
0.62772, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-300-1-1-latest.npy
84700 / 84710
0.62807, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-300-1-2-latest.npy
84700 / 84710
0.62806, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-300-1-3-latest.npy
84700 / 84710
0.62804, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-300-1-4-latest.npy
84700 / 84710
0.62699, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-300-1-5-latest.npy
84700 / 84710
0.62870, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-300-1-6-latest.npy
84700 / 84710
0.62928, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-1000-1-1-latest.npy
84700 / 84710
0.62505, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-1000-1-2-latest.npy
84700 / 84710
0.62692, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-1000-1-3-latest.npy
84700 / 84710
0.62831, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-1000-1-4-latest.npy
84700 / 84710
0.62934, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-1000-1-5-latest.npy
84700 / 84710
0.62769, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-0.3-1000-1-6-latest.npy
84700 / 84710
0.62969, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-0.01-1-1-latest.npy
84700 / 84710
0.64923, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-0.01-1-2-latest.npy
84700 / 84710
0.64971, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-0.01-1-3-latest.npy
84700 / 84710
0.64983, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-0.01-1-4-latest.npy
84700 / 84710
0.64935, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-0.01-1-5-latest.npy
84700 / 84710
0.64904, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-0.01-1-6-latest.npy
84700 / 84710
0.64954, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-0.1-1-1-latest.npy
84700 / 84710
0.64574, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-0.1-1-2-latest.npy
84700 / 84710
0.64596, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-0.1-1-3-latest.npy
84700 / 84710
0.64503, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-0.1-1-4-latest.npy
84700 / 84710
0.64626, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-0.1-1-5-latest.npy
84700 / 84710
0.64552, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-0.1-1-6-latest.npy
84700 / 84710
0.64585, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-1-1-1-latest.npy
84700 / 84710
0.63813, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-1-1-2-latest.npy
84700 / 84710
0.63861, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-1-1-3-latest.npy
84700 / 84710
0.63825, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-1-1-4-latest.npy
84700 / 84710
0.63810, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-1-1-5-latest.npy
84700 / 84710
0.63894, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-1-1-6-latest.npy
84700 / 84710
0.63827, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-3-1-1-latest.npy
84700 / 84710
0.63646, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-3-1-2-latest.npy
84700 / 84710
0.63542, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-3-1-3-latest.npy
84700 / 84710
0.63566, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-3-1-4-latest.npy
84700 / 84710
0.63546, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-3-1-5-latest.npy
84700 / 84710
0.63647, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-3-1-6-latest.npy
84700 / 84710
0.63520, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-10-1-1-latest.npy
84700 / 84710
0.63437, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-10-1-2-latest.npy
84700 / 84710
0.63495, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-10-1-3-latest.npy
84700 / 84710
0.63397, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-10-1-4-latest.npy
84700 / 84710
0.63476, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-10-1-5-latest.npy
84700 / 84710
0.63547, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-10-1-6-latest.npy
84700 / 84710
0.63557, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-30-1-1-latest.npy
84700 / 84710
0.63435, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-30-1-2-latest.npy
84700 / 84710
0.63529, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-30-1-3-latest.npy
84700 / 84710
0.63372, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-30-1-4-latest.npy
84700 / 84710
0.63544, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-30-1-5-latest.npy
84700 / 84710
0.63415, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-30-1-6-latest.npy
84700 / 84710
0.63495, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-100-1-1-latest.npy
84700 / 84710
0.63374, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-100-1-2-latest.npy
84700 / 84710
0.63500, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-100-1-3-latest.npy
84700 / 84710
0.63429, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-100-1-4-latest.npy
84700 / 84710
0.63364, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-100-1-5-latest.npy
84700 / 84710
0.63414, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-100-1-6-latest.npy
84700 / 84710
0.63419, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-300-1-1-latest.npy
84700 / 84710
0.63437, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-300-1-2-latest.npy
84700 / 84710
0.63387, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-300-1-3-latest.npy
84700 / 84710
0.63452, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-300-1-4-latest.npy
84700 / 84710
0.63475, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-300-1-5-latest.npy
84700 / 84710
0.63438, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-300-1-6-latest.npy
84700 / 84710
0.63486, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-1000-1-1-latest.npy
84700 / 84710
0.63467, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-1000-1-2-latest.npy
84700 / 84710
0.63449, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-1000-1-3-latest.npy
84700 / 84710
0.63421, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-1000-1-4-latest.npy
84700 / 84710
0.63473, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-1000-1-5-latest.npy
84700 / 84710
0.63401, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1-1000-1-6-latest.npy
84700 / 84710
0.63355, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-0.01-1-1-latest.npy
84700 / 84710
0.65065, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-0.01-1-2-latest.npy
84700 / 84710
0.65035, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-0.01-1-3-latest.npy
84700 / 84710
0.65132, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-0.01-1-4-latest.npy
84700 / 84710
0.65065, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-0.01-1-5-latest.npy
84700 / 84710
0.65085, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-0.01-1-6-latest.npy
84700 / 84710
0.65123, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-0.1-1-1-latest.npy
84700 / 84710
0.64736, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-0.1-1-2-latest.npy
84700 / 84710
0.64653, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-0.1-1-3-latest.npy
84700 / 84710
0.64692, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-0.1-1-4-latest.npy
84700 / 84710
0.64641, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-0.1-1-5-latest.npy
84700 / 84710
0.64651, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-0.1-1-6-latest.npy
84700 / 84710
0.64684, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-1-1-1-latest.npy
84700 / 84710
0.63930, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-1-1-2-latest.npy
84700 / 84710
0.63934, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-1-1-3-latest.npy
84700 / 84710
0.63824, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-1-1-4-latest.npy
84700 / 84710
0.63850, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-1-1-5-latest.npy
84700 / 84710
0.63871, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-1-1-6-latest.npy
84700 / 84710
0.63894, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-3-1-1-latest.npy
84700 / 84710
0.63652, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-3-1-2-latest.npy
84700 / 84710
0.63657, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-3-1-3-latest.npy
84700 / 84710
0.63655, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-3-1-4-latest.npy
84700 / 84710
0.63600, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-3-1-5-latest.npy
84700 / 84710
0.63675, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-3-1-6-latest.npy
84700 / 84710
0.63684, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-10-1-1-latest.npy
84700 / 84710
0.63527, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-10-1-2-latest.npy
84700 / 84710
0.63546, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-10-1-3-latest.npy
84700 / 84710
0.63545, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-10-1-4-latest.npy
84700 / 84710
0.63530, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-10-1-5-latest.npy
84700 / 84710
0.63542, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-10-1-6-latest.npy
84700 / 84710
0.63589, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-30-1-1-latest.npy
84700 / 84710
0.63479, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-30-1-2-latest.npy
84700 / 84710
0.63537, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-30-1-3-latest.npy
84700 / 84710
0.63499, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-30-1-4-latest.npy
84700 / 84710
0.63432, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-30-1-5-latest.npy
84700 / 84710
0.63532, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-30-1-6-latest.npy
84700 / 84710
0.63519, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-100-1-1-latest.npy
84700 / 84710
0.63492, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-100-1-2-latest.npy
84700 / 84710
0.63403, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-100-1-3-latest.npy
84700 / 84710
0.63506, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-100-1-4-latest.npy
84700 / 84710
0.63421, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-100-1-5-latest.npy
84700 / 84710
0.63463, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-100-1-6-latest.npy
84700 / 84710
0.63474, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-300-1-1-latest.npy
84700 / 84710
0.63538, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-300-1-2-latest.npy
84700 / 84710
0.63535, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-300-1-3-latest.npy
84700 / 84710
0.63509, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-300-1-4-latest.npy
84700 / 84710
0.63514, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-300-1-5-latest.npy
84700 / 84710
0.63492, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-300-1-6-latest.npy
84700 / 84710
0.63448, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-1000-1-1-latest.npy
84700 / 84710
0.63428, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-1000-1-2-latest.npy
84700 / 84710
0.63460, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-1000-1-3-latest.npy
84700 / 84710
0.63392, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-1000-1-4-latest.npy
84700 / 84710
0.63474, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-1000-1-5-latest.npy
84700 / 84710
0.63489, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-3-1000-1-6-latest.npy
84700 / 84710
0.63451, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-0.01-1-1-latest.npy
84700 / 84710
0.65060, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-0.01-1-2-latest.npy
84700 / 84710
0.65116, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-0.01-1-3-latest.npy
84700 / 84710
0.65003, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-0.01-1-4-latest.npy
84700 / 84710
0.64986, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-0.01-1-5-latest.npy
84700 / 84710
0.65024, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-0.01-1-6-latest.npy
84700 / 84710
0.65016, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-0.1-1-1-latest.npy
84700 / 84710
0.64676, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-0.1-1-2-latest.npy
84700 / 84710
0.64627, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-0.1-1-3-latest.npy
84700 / 84710
0.64723, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-0.1-1-4-latest.npy
84700 / 84710
0.64703, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-0.1-1-5-latest.npy
84700 / 84710
0.64774, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-0.1-1-6-latest.npy
84700 / 84710
0.64614, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-1-1-1-latest.npy
84700 / 84710
0.63854, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-1-1-2-latest.npy
84700 / 84710
0.63924, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-1-1-3-latest.npy
84700 / 84710
0.63906, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-1-1-4-latest.npy
84700 / 84710
0.63867, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-1-1-5-latest.npy
84700 / 84710
0.63854, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-1-1-6-latest.npy
84700 / 84710
0.63816, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-3-1-1-latest.npy
84700 / 84710
0.63652, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-3-1-2-latest.npy
84700 / 84710
0.63688, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-3-1-3-latest.npy
84700 / 84710
0.63661, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-3-1-4-latest.npy
84700 / 84710
0.63739, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-3-1-5-latest.npy
84700 / 84710
0.63651, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-3-1-6-latest.npy
84700 / 84710
0.63593, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-10-1-1-latest.npy
84700 / 84710
0.63510, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-10-1-2-latest.npy
84700 / 84710
0.63557, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-10-1-3-latest.npy
84700 / 84710
0.63506, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-10-1-4-latest.npy
84700 / 84710
0.63550, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-10-1-5-latest.npy
84700 / 84710
0.63502, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-10-1-6-latest.npy
84700 / 84710
0.63541, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-30-1-1-latest.npy
84700 / 84710
0.63441, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-30-1-2-latest.npy
84700 / 84710
0.63528, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-30-1-3-latest.npy
84700 / 84710
0.63529, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-30-1-4-latest.npy
84700 / 84710
0.63514, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-30-1-5-latest.npy
84700 / 84710
0.63479, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-30-1-6-latest.npy
84700 / 84710
0.63540, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-100-1-1-latest.npy
84700 / 84710
0.63461, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-100-1-2-latest.npy
84700 / 84710
0.63445, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-100-1-3-latest.npy
84700 / 84710
0.63472, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-100-1-4-latest.npy
84700 / 84710
0.63498, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-100-1-5-latest.npy
84700 / 84710
0.63496, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-100-1-6-latest.npy
84700 / 84710
0.63459, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-300-1-1-latest.npy
84700 / 84710
0.63498, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-300-1-2-latest.npy
84700 / 84710
0.63519, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-300-1-3-latest.npy
84700 / 84710
0.63460, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-300-1-4-latest.npy
84700 / 84710
0.63465, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-300-1-5-latest.npy
84700 / 84710
0.63470, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-300-1-6-latest.npy
84700 / 84710
0.63452, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-1000-1-1-latest.npy
84700 / 84710
0.63404, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-1000-1-2-latest.npy
84700 / 84710
0.63470, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-1000-1-3-latest.npy
84700 / 84710
0.63468, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-1000-1-4-latest.npy
84700 / 84710
0.63475, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-1000-1-5-latest.npy
84700 / 84710
0.63487, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-10-1000-1-6-latest.npy
84700 / 84710
0.63522, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-0.01-1-1-latest.npy
84700 / 84710
0.65119, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-0.01-1-2-latest.npy
84700 / 84710
0.65045, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-0.01-1-3-latest.npy
84700 / 84710
0.65088, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-0.01-1-4-latest.npy
84700 / 84710
0.65038, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-0.01-1-5-latest.npy
84700 / 84710
0.65068, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-0.01-1-6-latest.npy
84700 / 84710
0.65111, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-0.1-1-1-latest.npy
84700 / 84710
0.64716, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-0.1-1-2-latest.npy
84700 / 84710
0.64708, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-0.1-1-3-latest.npy
84700 / 84710
0.64670, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-0.1-1-4-latest.npy
84700 / 84710
0.64693, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-0.1-1-5-latest.npy
84700 / 84710
0.64707, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-0.1-1-6-latest.npy
84700 / 84710
0.64698, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-1-1-1-latest.npy
84700 / 84710
0.63940, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-1-1-2-latest.npy
84700 / 84710
0.63872, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-1-1-3-latest.npy
84700 / 84710
0.63926, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-1-1-4-latest.npy
84700 / 84710
0.63867, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-1-1-5-latest.npy
84700 / 84710
0.63877, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-1-1-6-latest.npy
84700 / 84710
0.63786, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-3-1-1-latest.npy
84700 / 84710
0.63601, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-3-1-2-latest.npy
84700 / 84710
0.63684, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-3-1-3-latest.npy
84700 / 84710
0.63631, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-3-1-4-latest.npy
84700 / 84710
0.63705, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-3-1-5-latest.npy
84700 / 84710
0.63692, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-3-1-6-latest.npy
84700 / 84710
0.63696, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-10-1-1-latest.npy
84700 / 84710
0.63538, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-10-1-2-latest.npy
84700 / 84710
0.63621, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-10-1-3-latest.npy
84700 / 84710
0.63552, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-10-1-4-latest.npy
84700 / 84710
0.63662, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-10-1-5-latest.npy
84700 / 84710
0.63495, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-10-1-6-latest.npy
84700 / 84710
0.63520, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-30-1-1-latest.npy
84700 / 84710
0.63554, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-30-1-2-latest.npy
84700 / 84710
0.63445, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-30-1-3-latest.npy
84700 / 84710
0.63426, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-30-1-4-latest.npy
84700 / 84710
0.63496, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-30-1-5-latest.npy
84700 / 84710
0.63530, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-30-1-6-latest.npy
84700 / 84710
0.63576, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-100-1-1-latest.npy
84700 / 84710
0.63509, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-100-1-2-latest.npy
84700 / 84710
0.63494, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-100-1-3-latest.npy
84700 / 84710
0.63440, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-100-1-4-latest.npy
84700 / 84710
0.63480, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-100-1-5-latest.npy
84700 / 84710
0.63413, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-100-1-6-latest.npy
84700 / 84710
0.63486, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-300-1-1-latest.npy
84700 / 84710
0.63444, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-300-1-2-latest.npy
84700 / 84710
0.63463, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-300-1-3-latest.npy
84700 / 84710
0.63487, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-300-1-4-latest.npy
84700 / 84710
0.63449, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-300-1-5-latest.npy
84700 / 84710
0.63449, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-300-1-6-latest.npy
84700 / 84710
0.63436, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-1000-1-1-latest.npy
84700 / 84710
0.63430, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-1000-1-2-latest.npy
84700 / 84710
0.63545, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-1000-1-3-latest.npy
84700 / 84710
0.63529, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-1000-1-4-latest.npy
84700 / 84710
0.63521, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-1000-1-5-latest.npy
84700 / 84710
0.63475, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-30-1000-1-6-latest.npy
84700 / 84710
0.63434, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-0.01-1-1-latest.npy
84700 / 84710
0.65018, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-0.01-1-2-latest.npy
84700 / 84710
0.65023, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-0.01-1-3-latest.npy
84700 / 84710
0.65070, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-0.01-1-4-latest.npy
84700 / 84710
0.65006, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-0.01-1-5-latest.npy
84700 / 84710
0.65120, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-0.01-1-6-latest.npy
84700 / 84710
0.65082, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-0.1-1-1-latest.npy
84700 / 84710
0.64661, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-0.1-1-2-latest.npy
84700 / 84710
0.64728, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-0.1-1-3-latest.npy
84700 / 84710
0.64738, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-0.1-1-4-latest.npy
84700 / 84710
0.64693, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-0.1-1-5-latest.npy
84700 / 84710
0.64721, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-0.1-1-6-latest.npy
84700 / 84710
0.64700, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-1-1-1-latest.npy
84700 / 84710
0.63911, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-1-1-2-latest.npy
84700 / 84710
0.63849, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-1-1-3-latest.npy
84700 / 84710
0.63846, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-1-1-4-latest.npy
84700 / 84710
0.63865, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-1-1-5-latest.npy
84700 / 84710
0.63814, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-1-1-6-latest.npy
84700 / 84710
0.63847, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-3-1-1-latest.npy
84700 / 84710
0.63602, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-3-1-2-latest.npy
84700 / 84710
0.63612, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-3-1-3-latest.npy
84700 / 84710
0.63711, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-3-1-4-latest.npy
84700 / 84710
0.63723, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-3-1-5-latest.npy
84700 / 84710
0.63607, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-3-1-6-latest.npy
84700 / 84710
0.63602, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-10-1-1-latest.npy
84700 / 84710
0.63549, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-10-1-2-latest.npy
84700 / 84710
0.63560, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-10-1-3-latest.npy
84700 / 84710
0.63528, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-10-1-4-latest.npy
84700 / 84710
0.63618, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-10-1-5-latest.npy
84700 / 84710
0.63591, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-10-1-6-latest.npy
84700 / 84710
0.63559, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-30-1-1-latest.npy
84700 / 84710
0.63524, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-30-1-2-latest.npy
84700 / 84710
0.63475, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-30-1-3-latest.npy
84700 / 84710
0.63469, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-30-1-4-latest.npy
84700 / 84710
0.63438, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-30-1-5-latest.npy
84700 / 84710
0.63486, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-30-1-6-latest.npy
84700 / 84710
0.63546, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-100-1-1-latest.npy
84700 / 84710
0.63439, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-100-1-2-latest.npy
84700 / 84710
0.63424, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-100-1-3-latest.npy
84700 / 84710
0.63567, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-100-1-4-latest.npy
84700 / 84710
0.63475, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-100-1-5-latest.npy
84700 / 84710
0.63484, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-100-1-6-latest.npy
84700 / 84710
0.63551, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-300-1-1-latest.npy
84700 / 84710
0.63477, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-300-1-2-latest.npy
84700 / 84710
0.63525, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-300-1-3-latest.npy
84700 / 84710
0.63445, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-300-1-4-latest.npy
84700 / 84710
0.63493, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-300-1-5-latest.npy
84700 / 84710
0.63533, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-300-1-6-latest.npy
84700 / 84710
0.63509, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-1000-1-1-latest.npy
84700 / 84710
0.63541, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-1000-1-2-latest.npy
84700 / 84710
0.63445, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-1000-1-3-latest.npy
84700 / 84710
0.63466, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-1000-1-4-latest.npy
84700 / 84710
0.63497, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-1000-1-5-latest.npy
84700 / 84710
0.63510, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-100-1000-1-6-latest.npy
84700 / 84710
0.63541, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-0.01-1-1-latest.npy
84700 / 84710
0.65046, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-0.01-1-2-latest.npy
84700 / 84710
0.65089, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-0.01-1-3-latest.npy
84700 / 84710
0.65038, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-0.01-1-4-latest.npy
84700 / 84710
0.65065, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-0.01-1-5-latest.npy
84700 / 84710
0.65072, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-0.01-1-6-latest.npy
84700 / 84710
0.65101, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-0.1-1-1-latest.npy
84700 / 84710
0.64694, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-0.1-1-2-latest.npy
84700 / 84710
0.64718, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-0.1-1-3-latest.npy
84700 / 84710
0.64649, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-0.1-1-4-latest.npy
84700 / 84710
0.64692, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-0.1-1-5-latest.npy
84700 / 84710
0.64702, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-0.1-1-6-latest.npy
84700 / 84710
0.64738, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-1-1-1-latest.npy
84700 / 84710
0.63890, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-1-1-2-latest.npy
84700 / 84710
0.63896, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-1-1-3-latest.npy
84700 / 84710
0.63873, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-1-1-4-latest.npy
84700 / 84710
0.63904, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-1-1-5-latest.npy
84700 / 84710
0.63882, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-1-1-6-latest.npy
84700 / 84710
0.63924, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-3-1-1-latest.npy
84700 / 84710
0.63724, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-3-1-2-latest.npy
84700 / 84710
0.63654, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-3-1-3-latest.npy
84700 / 84710
0.63653, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-3-1-4-latest.npy
84700 / 84710
0.63608, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-3-1-5-latest.npy
84700 / 84710
0.63632, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-3-1-6-latest.npy
84700 / 84710
0.63679, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-10-1-1-latest.npy
84700 / 84710
0.63540, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-10-1-2-latest.npy
84700 / 84710
0.63515, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-10-1-3-latest.npy
84700 / 84710
0.63595, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-10-1-4-latest.npy
84700 / 84710
0.63555, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-10-1-5-latest.npy
84700 / 84710
0.63460, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-10-1-6-latest.npy
84700 / 84710
0.63446, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-30-1-1-latest.npy
84700 / 84710
0.63481, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-30-1-2-latest.npy
84700 / 84710
0.63470, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-30-1-3-latest.npy
84700 / 84710
0.63517, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-30-1-4-latest.npy
84700 / 84710
0.63461, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-30-1-5-latest.npy
84700 / 84710
0.63469, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-30-1-6-latest.npy
84700 / 84710
0.63437, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-100-1-1-latest.npy
84700 / 84710
0.63481, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-100-1-2-latest.npy
84700 / 84710
0.63526, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-100-1-3-latest.npy
84700 / 84710
0.63514, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-100-1-4-latest.npy
84700 / 84710
0.63538, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-100-1-5-latest.npy
84700 / 84710
0.63456, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-100-1-6-latest.npy
84700 / 84710
0.63510, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-300-1-1-latest.npy
84700 / 84710
0.63482, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-300-1-2-latest.npy
84700 / 84710
0.63497, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-300-1-3-latest.npy
84700 / 84710
0.63447, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-300-1-4-latest.npy
84700 / 84710
0.63571, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-300-1-5-latest.npy
84700 / 84710
0.63511, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-300-1-6-latest.npy
84700 / 84710
0.63511, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-1000-1-1-latest.npy
84700 / 84710
0.63504, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-1000-1-2-latest.npy
84700 / 84710
0.63489, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-1000-1-3-latest.npy
84700 / 84710
0.63537, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-1000-1-4-latest.npy
84700 / 84710
0.63550, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-1000-1-5-latest.npy
84700 / 84710
0.63478, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-300-1000-1-6-latest.npy
84700 / 84710
0.63391, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-0.01-1-1-latest.npy
84700 / 84710
0.65131, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-0.01-1-2-latest.npy
84700 / 84710
0.65063, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-0.01-1-3-latest.npy
84700 / 84710
0.65133, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-0.01-1-4-latest.npy
84700 / 84710
0.65110, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-0.01-1-5-latest.npy
84700 / 84710
0.65056, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-0.01-1-6-latest.npy
84700 / 84710
0.65039, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-0.1-1-1-latest.npy
84700 / 84710
0.64692, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-0.1-1-2-latest.npy
84700 / 84710
0.64710, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-0.1-1-3-latest.npy
84700 / 84710
0.64711, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-0.1-1-4-latest.npy
84700 / 84710
0.64717, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-0.1-1-5-latest.npy
84700 / 84710
0.64714, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-0.1-1-6-latest.npy
84700 / 84710
0.64627, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-1-1-1-latest.npy
84700 / 84710
0.63819, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-1-1-2-latest.npy
84700 / 84710
0.63945, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-1-1-3-latest.npy
84700 / 84710
0.63936, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-1-1-4-latest.npy
84700 / 84710
0.63934, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-1-1-5-latest.npy
84700 / 84710
0.63877, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-1-1-6-latest.npy
84700 / 84710
0.63881, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-3-1-1-latest.npy
84700 / 84710
0.63710, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-3-1-2-latest.npy
84700 / 84710
0.63570, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-3-1-3-latest.npy
84700 / 84710
0.63618, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-3-1-4-latest.npy
84700 / 84710
0.63628, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-3-1-5-latest.npy
84700 / 84710
0.63624, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-3-1-6-latest.npy
84700 / 84710
0.63607, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-10-1-1-latest.npy
84700 / 84710
0.63524, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-10-1-2-latest.npy
84700 / 84710
0.63461, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-10-1-3-latest.npy
84700 / 84710
0.63498, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-10-1-4-latest.npy
84700 / 84710
0.63636, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-10-1-5-latest.npy
84700 / 84710
0.63544, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-10-1-6-latest.npy
84700 / 84710
0.63478, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-30-1-1-latest.npy
84700 / 84710
0.63541, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-30-1-2-latest.npy
84700 / 84710
0.63527, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-30-1-3-latest.npy
84700 / 84710
0.63511, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-30-1-4-latest.npy
84700 / 84710
0.63531, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-30-1-5-latest.npy
84700 / 84710
0.63474, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-30-1-6-latest.npy
84700 / 84710
0.63478, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-100-1-1-latest.npy
84700 / 84710
0.63474, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-100-1-2-latest.npy
84700 / 84710
0.63487, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-100-1-3-latest.npy
84700 / 84710
0.63456, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-100-1-4-latest.npy
84700 / 84710
0.63584, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-100-1-5-latest.npy
84700 / 84710
0.63505, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-100-1-6-latest.npy
84700 / 84710
0.63489, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-300-1-1-latest.npy
84700 / 84710
0.63427, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-300-1-2-latest.npy
84700 / 84710
0.63476, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-300-1-3-latest.npy
84700 / 84710
0.63490, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-300-1-4-latest.npy
84700 / 84710
0.63455, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-300-1-5-latest.npy
84700 / 84710
0.63468, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-300-1-6-latest.npy
84700 / 84710
0.63507, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-1000-1-1-latest.npy
84700 / 84710
0.63503, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-1000-1-2-latest.npy
84700 / 84710
0.63431, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-1000-1-3-latest.npy
84700 / 84710
0.63481, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-1000-1-4-latest.npy
84700 / 84710
0.63482, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-1000-1-5-latest.npy
84700 / 84710
0.63486, 37886 / 84710

data/aotm-2011/setting1/mlr_both/mlr-N-both-1000-1000-1-6-latest.npy
84700 / 84710
0.63524, 37886 / 84710

samples + multitask regularisation


In [17]:
D = X_train.shape[1]
K = Y_train.shape[1]
for c1 in C1:
    for c3 in C3:
        for p in P:
            fname = os.path.join(data_dir, 'mlr_multitask/mlr-Y-samples-%g-1-%g-%d-latest.npy' % (c1, c3, p))
            if not os.path.exists(fname):continue
            print(fname)
            w = np.load(fname)
            aucs = calc_auc_w(W=w[1:].reshape(K, D), b=w[0], X=X_dev, Y=Y_dev)
            print('\n%.5f, %d / %d\n' % (np.mean(aucs), len(aucs), Y_dev.shape[1]))


data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.01-1-0.01-3-latest.npy
84700 / 84710
0.48678, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.01-1-0.1-3-latest.npy
84700 / 84710
0.49614, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.01-1-0.3-3-latest.npy
84700 / 84710
0.48040, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.01-1-1-3-latest.npy
84700 / 84710
0.50120, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.01-1-10-3-latest.npy
84700 / 84710
0.47006, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.01-1-30-3-latest.npy
84700 / 84710
0.49813, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.01-1-100-3-latest.npy
84700 / 84710
0.50232, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.01-1-300-3-latest.npy
84700 / 84710
0.51547, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.01-1-1000-3-latest.npy
84700 / 84710
0.49787, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.03-1-0.01-3-latest.npy
84700 / 84710
0.59715, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.03-1-0.1-3-latest.npy
84700 / 84710
0.59370, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.03-1-0.3-3-latest.npy
84700 / 84710
0.59085, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.03-1-1-3-latest.npy
84700 / 84710
0.60162, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.03-1-10-3-latest.npy
84700 / 84710
0.58287, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.03-1-30-3-latest.npy
84700 / 84710
0.58370, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.03-1-100-3-latest.npy
84700 / 84710
0.58295, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.03-1-300-3-latest.npy
84700 / 84710
0.58575, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.03-1-1000-3-latest.npy
84700 / 84710
0.58524, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.1-1-0.01-3-latest.npy
84700 / 84710
0.59838, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.1-1-0.1-3-latest.npy
84700 / 84710
0.60112, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.1-1-0.3-3-latest.npy
84700 / 84710
0.59669, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.1-1-1-3-latest.npy
84700 / 84710
0.60148, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.1-1-10-3-latest.npy
84700 / 84710
0.60117, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.1-1-30-3-latest.npy
84700 / 84710
0.60068, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.1-1-100-3-latest.npy
84700 / 84710
0.60635, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.1-1-300-3-latest.npy
84700 / 84710
0.60021, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.1-1-1000-3-latest.npy
84700 / 84710
0.59464, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.3-1-0.01-3-latest.npy
84700 / 84710
0.63823, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.3-1-0.1-3-latest.npy
84700 / 84710
0.63532, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.3-1-0.3-3-latest.npy
84700 / 84710
0.63582, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.3-1-1-3-latest.npy
84700 / 84710
0.63454, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.3-1-10-3-latest.npy
84700 / 84710
0.63665, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.3-1-30-3-latest.npy
84700 / 84710
0.63680, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.3-1-100-3-latest.npy
84700 / 84710
0.63891, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.3-1-300-3-latest.npy
84700 / 84710
0.63609, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-0.3-1-1000-3-latest.npy
84700 / 84710
0.63684, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-1-1-0.01-3-latest.npy
84700 / 84710
0.64922, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-1-1-0.1-3-latest.npy
84700 / 84710
0.64830, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-1-1-0.3-3-latest.npy
84700 / 84710
0.64935, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-1-1-1-3-latest.npy
84700 / 84710
0.64915, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-1-1-10-3-latest.npy
84700 / 84710
0.64934, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-1-1-30-3-latest.npy
84700 / 84710
0.64874, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-1-1-100-3-latest.npy
84700 / 84710
0.64848, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-1-1-300-3-latest.npy
84700 / 84710
0.64932, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-1-1-1000-3-latest.npy
84700 / 84710
0.64885, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-3-1-0.01-3-latest.npy
84700 / 84710
0.65131, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-3-1-0.1-3-latest.npy
84700 / 84710
0.65076, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-3-1-0.3-3-latest.npy
84700 / 84710
0.65060, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-3-1-1-3-latest.npy
84700 / 84710
0.64987, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-3-1-10-3-latest.npy
84700 / 84710
0.64947, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-3-1-30-3-latest.npy
84700 / 84710
0.64943, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-3-1-100-3-latest.npy
84700 / 84710
0.64973, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-3-1-300-3-latest.npy
84700 / 84710
0.65007, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-3-1-1000-3-latest.npy
84700 / 84710
0.65004, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-10-1-0.01-3-latest.npy
84700 / 84710
0.65310, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-10-1-0.1-3-latest.npy
84700 / 84710
0.65092, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-10-1-0.3-3-latest.npy
84700 / 84710
0.65157, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-10-1-1-3-latest.npy
84700 / 84710
0.65116, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-10-1-10-3-latest.npy
84700 / 84710
0.65029, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-10-1-30-3-latest.npy
84700 / 84710
0.65006, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-10-1-100-3-latest.npy
84700 / 84710
0.65067, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-10-1-300-3-latest.npy
84700 / 84710
0.65022, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-10-1-1000-3-latest.npy
84700 / 84710
0.65058, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-30-1-0.01-3-latest.npy
84700 / 84710
0.65347, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-30-1-0.1-3-latest.npy
84700 / 84710
0.65174, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-30-1-0.3-3-latest.npy
84700 / 84710
0.65148, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-30-1-1-3-latest.npy
84700 / 84710
0.65078, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-30-1-10-3-latest.npy
84700 / 84710
0.65013, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-30-1-30-3-latest.npy
84700 / 84710
0.65019, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-30-1-100-3-latest.npy
84700 / 84710
0.65057, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-30-1-300-3-latest.npy
84700 / 84710
0.65066, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-30-1-1000-3-latest.npy
84700 / 84710
0.64959, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-100-1-0.01-3-latest.npy
84700 / 84710
0.65418, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-100-1-0.1-3-latest.npy
84700 / 84710
0.65151, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-100-1-0.3-3-latest.npy
84700 / 84710
0.65057, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-100-1-1-3-latest.npy
84700 / 84710
0.65087, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-100-1-10-3-latest.npy
84700 / 84710
0.65147, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-100-1-30-3-latest.npy
84700 / 84710
0.65064, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-100-1-100-3-latest.npy
84700 / 84710
0.65056, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-100-1-300-3-latest.npy
84700 / 84710
0.65080, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-100-1-1000-3-latest.npy
84700 / 84710
0.65006, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-300-1-0.01-3-latest.npy
84700 / 84710
0.65420, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-300-1-0.1-3-latest.npy
84700 / 84710
0.65206, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-300-1-0.3-3-latest.npy
84700 / 84710
0.65106, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-300-1-1-3-latest.npy
84700 / 84710
0.65140, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-300-1-10-3-latest.npy
84700 / 84710
0.65033, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-300-1-30-3-latest.npy
84700 / 84710
0.65038, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-300-1-100-3-latest.npy
84700 / 84710
0.65057, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-300-1-300-3-latest.npy
84700 / 84710
0.65092, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-300-1-1000-3-latest.npy
84700 / 84710
0.65041, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-1000-1-0.01-3-latest.npy
84700 / 84710
0.65378, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-1000-1-0.1-3-latest.npy
84700 / 84710
0.65205, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-1000-1-0.3-3-latest.npy
84700 / 84710
0.65151, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-1000-1-1-3-latest.npy
84700 / 84710
0.65023, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-1000-1-10-3-latest.npy
84700 / 84710
0.64998, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-1000-1-30-3-latest.npy
84700 / 84710
0.65027, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-1000-1-100-3-latest.npy
84700 / 84710
0.64974, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-1000-1-300-3-latest.npy
84700 / 84710
0.65083, 37886 / 84710

data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-1000-1-1000-3-latest.npy
84700 / 84710
0.65020, 37886 / 84710


In [23]:
print('Choose:', 'data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-100-1-0.01-3-latest.npy')


Choose: data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-100-1-0.01-3-latest.npy

In [24]:
print('Choose:', 'data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-300-1-0.01-3-latest.npy')


Choose: data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-300-1-0.01-3-latest.npy

MLR on test set


In [27]:
perf_mlr = {'aotm2011': {'Test': dict()}}
fperf_mlr = os.path.join(data_dir, 'perf_mlr.pkl')

loss weighting: samples


In [28]:
for fname in ['mlr-N-samples-10-1-1-3-latest.npy', 'mlr-N-samples-30-1-1-1-latest.npy']:
    fname = os.path.join(data_dir, 'mlr_train_dev/' + fname)
    print(fname)
    w = np.load(fname)
    aucs = calc_auc_w(W=w[1:].reshape(K, D), b=w[0], X=X_test, Y=Y_test)
    print('\n%.5f, %d / %d\n' % (np.mean(aucs), len(aucs), Y_test.shape[1]))
    mean_hitrates, ncols = calc_hitrate_w(W=w[1:].reshape(K, D), b=w[0], X=X_test, Y=Y_test, tops=TOPs)
    #print(mean_hitrates, ncols)
    break
perf_mlr['aotm2011']['Test']['samples'] = {'AUC': np.mean(aucs), 'HitRate': mean_hitrates}


data/aotm-2011/setting1/mlr_train_dev/mlr-N-samples-10-1-1-3-latest.npy
84700 / 84710
0.66880, 52797 / 84710

84700 / 84710[0.0005173686433985314, 0.0015332700016488867, 0.0039052728843341953, 0.005827025456691915, 0.009636483534858444, 0.018033476473586704, 0.03315738437620363, 0.04693657782114512, 0.0731851652048598, 0.12943044407862983] 52797

In [30]:
perf_mlr


Out[30]:
{'aotm2011': {'Test': {'samples': {'AUC': 0.6687959160672104,
    'HitRate': [0.0005173686433985314,
     0.0015332700016488867,
     0.0039052728843341953,
     0.005827025456691915,
     0.009636483534858444,
     0.018033476473586704,
     0.03315738437620363,
     0.04693657782114512,
     0.0731851652048598,
     0.12943044407862983]}}}}

loss weighting: labels


In [ ]:
fname = os.path.join(data_dir, 'mlr_train_dev/mlr-N-labels-30-1-1-3-latest.npy')
print(fname)
w = np.load(fname)
aucs = calc_auc_w(W=w[1:].reshape(K, D), b=w[0], X=X_test, Y=Y_test)
print('\n%.5f, %d / %d\n' % (np.mean(aucs), len(aucs), Y_test.shape[1]))
mean_hitrates, ncols = calc_hitrate_w(W=w[1:].reshape(K, D), b=w[0], X=X_test, Y=Y_test, tops=TOPs)
#print(mean_hitrates, ncols)
perf_mlr['aotm2011']['Test']['labels'] = {'AUC': np.mean(aucs), 'HitRate': mean_hitrates}


data/aotm-2011/setting1/mlr_train_dev/mlr-N-labels-30-1-1-3-latest.npy
84700 / 84710
0.64620, 52797 / 84710

84700 / 84710

loss weighting: both


In [ ]:
fname = os.path.join(data_dir, 'mlr_train_dev/mlr-N-both-3-0.01-1-3-latest.npy')
print(fname)
w = np.load(fname)
aucs = calc_auc_w(W=w[1:].reshape(K, D), b=w[0], X=X_test, Y=Y_test)
print('\n%.5f, %d / %d\n' % (np.mean(aucs), len(aucs), Y_test.shape[1]))
mean_hitrates, ncols = calc_hitrate_w(W=w[1:].reshape(K, D), b=w[0], X=X_test, Y=Y_test, tops=TOPs)
#print(mean_hitrates, ncols)
perf_mlr['aotm2011']['Test']['both'] = {'AUC': np.mean(aucs), 'HitRate': mean_hitrates}


data/aotm-2011/setting1/mlr_train_dev/mlr-N-both-3-0.01-1-3-latest.npy
7100 / 84710

loss weighting best (samples) + multitask regularisation


In [ ]:
fname = os.path.join(data_dir, '')
print(fname)
w = np.load(fname)
aucs = calc_auc_w(W=w[1:].reshape(K, D), b=w[0], X=X_test, Y=Y_test)
print('\n%.5f, %d / %d\n' % (np.mean(aucs), len(aucs), Y_test.shape[1]))
mean_hitrates, ncols = calc_hitrate_w(W=w[1:].reshape(K, D), b=w[0], X=X_test, Y=Y_test, tops=TOPs)
#print(mean_hitrates, ncols)
perf_mlr['aotm2011']['Test']['multitask'] = {'AUC': np.mean(aucs), 'HitRate': mean_hitrates}

BR for new song recommendation


In [35]:
base_dir = 'data/aotm-2011/setting1'
br_name = 'br1'
br_dir = os.path.join(base_dir, br_name)
fsplit = os.path.join(br_dir, br_name + '.split')

In [ ]:
X_test  = pkl.load(gzip.open(os.path.join(base_dir, 'X_test.pkl.gz'), 'rb'))
Y_test  = pkl.load(gzip.open(os.path.join(base_dir, 'Y_test.pkl.gz'), 'rb'))

In [ ]:
aucs = []
with open(fsplit, 'r') as fd:
    for line in fd:
        start, end = line.strip().split(' ')
        print(start, end)
        fname = os.path.join(br_dir, 'br1-aotm2011-%s-%s.pkl' % (start, end))
        #print(fname)
        #break
        br = pkl.load(open(fname, 'rb'))
        pred = br.predict(X_test)
        Y_part = Y_test[:, int(start):int(end)]
        nzcol = np.nonzero(np.sum(Y_part, axis=0))[1]
        #print(np.sum(Y_part, axis=0))
        #print('-'*20)
        #print(nzcol)
        auc, npl = calc_auc_pl(Y_part[:, nzcol].toarray(), pred[:, nzcol])
        aucs.append((auc, npl))

In [ ]:
mean_auc = np.sum([np.prod(t) for t in aucs]) / np.sum([t[1] for t in aucs])
print(mean_auc)

Hit rate curve.


In [ ]:
Y_test.shape

In [ ]:
hitrates = []  # [(hitrates, ncols)]
with open(fsplit, 'r') as fd:
    for line in fd:
        start, end = line.strip().split(' ')
        print(start, end)
        fname = os.path.join(br_dir, 'br1-aotm2011-%s-%s.pkl' % (start, end))
        br = pkl.load(open(fname, 'rb'))
        pred = br.predict(X_test)
        Y_part = Y_test[:, int(start):int(end)]
        hitrates.append(calc_hitrate_pl(Y_part, pred, tops=TOPs))

In [ ]:
mean_hitrate_dict = dict()
denom = np.sum([t[1] for t in hitrates])
for i in range(len(tops)):
    mean_hitrate_dict[tops[i]] = np.sum([t[0][i] * t[1] for t in hitrates]) / denom
#print(mean_hitrate_dict)

In [ ]:
xx = sorted(mean_hitrate_dict.keys())
yy = [mean_hitrate_dict[top] for top in xx]
ax = plt.subplot(111)
ax.plot(xx, yy, ls='--', marker='o')
ax.set_xlabel('Top-N')
ax.set_ylabel('Hit Rate')
ax.set_xscale('log')
#ax.set_xlim(0, 500)

In [ ]:
br1_perf = {'aotm2011': {'Test': {'AUC': mean_auc, 'HitRate': mean_hitrate_dict}}}
br1_perf

In [36]:
fperf = os.path.join(base_dir, 'perf-' + br_name + '.pkl')
#pkl.dump(br1_perf, open(fperf, 'wb'))
pkl.load(open(fperf, 'rb'))


Out[36]:
{'aotm2011': {'Test': {'AUC': 0.6126755471495918,
   'HitRate': {5: 0.0014351505245874243,
    10: 0.0032520941954716606,
    20: 0.005910632368811811,
    30: 0.008263519625670885,
    50: 0.012301003887173556,
    100: 0.02061297801493865,
    200: 0.03507022705138029,
    300: 0.04729167496261769,
    500: 0.0699152500877554,
    1000: 0.113767123918867,
    3000: 0.25071657862090035,
    10000: 0.5947338565933447,
    22886: 1.0}}}}

BR for playlist augmentation


In [37]:
base_dir = 'data/aotm-2011/setting2'
br_name = 'br2'
br_dir = os.path.join(base_dir, br_name)
fsplit = os.path.join(br_dir, br_name + '.split')

In [12]:
X = pkl.load(gzip.open(os.path.join(base_dir, 'X_train.pkl.gz'), 'rb'))
Y = pkl.load(gzip.open(os.path.join(base_dir, 'Y.pkl.gz'), 'rb'))
PU_test = pkl.load(gzip.open(os.path.join(base_dir, 'PU_test.pkl.gz'), 'rb'))

In [9]:
print(Y.shape)
print(PU_test.shape)


(114428, 84710)
(114428, 16942)

In [10]:
np.sum(PU_test, axis=0)


Out[10]:
matrix([[3, 5, 3, ..., 6, 6, 5]], dtype=int64)

In [11]:
Y_test = Y[:, -PU_test.shape[1]:]
print(Y_test.shape)
Y_test.sum(axis=0)


(114428, 16942)
Out[11]:
matrix([[ 6, 10,  6, ..., 12, 11, 10]], dtype=int64)

In [ ]:
np.where(PU_test[:,0:10].toarray()[:,0] == 0)

In [ ]:
aucs_pla = []
hitrates_pla = {top: [] for top in TOPs}
with open(fsplit, 'r') as fd:
    for line in fd:
        start, end = line.strip().split(' ')
        print(start, end)
        fname = os.path.join(br_dir, br_name + '-aotm2011-%s-%s.pkl' % (start, end))
        br = pkl.load(open(fname, 'rb'))
        Y_pred = br.predict(X)
        Y_part = Y_test[:, int(start):int(end)]
        PU_part = PU_test[:, int(start):int(end)]
        assert Y_part.shape == PU_part.shape == Y_pred.shape
        if issparse(Y_part):
            Y_part = Y_part.toarray()
        if issparse(PU_part):
            PU_part = PU_part.toarray()
        for j in range(Y_part.shape[1]):
            indices = np.where(0 == PU_part[:, j])[0]
            y_true = Y_part[indices, j].reshape(-1)
            y_pred = Y_pred[indices, j].reshape(-1)
            #print(y_true.shape)
            #print(y_pred.shape)
            
            # auc
            aucs_pla.append(roc_auc_score(y_true, y_pred))
            
            # hitrates
            sortix = np.argsort(-y_pred)
            npos = np.sum(y_true)
            for top in TOPs:
                topix = sortix[:top]
                hitrates_pla[top].append(np.sum(y_true[topix]) / npos)

In [ ]:
mean_auc_pla = np.mean(aucs_pla)
print(mean_auc_pla)

In [ ]:
#aucs_pla

In [ ]:
hitrates_pla.keys()

In [ ]:
mean_hitrates = {top: np.mean(hitrates_pla[top]) for top in hitrates_pla}
#mean_hitrates

In [ ]:
br2_perf = {'aotm2011': {'Test': {'AUC': mean_auc_pla, 'HitRate': mean_hitrates}}}
br2_perf

In [38]:
fperf_pla = os.path.join(base_dir, 'perf-' + br_name + '.pkl')
print(fperf_pla)
#pkl.dump(br2_perf, open(fperf_pla, 'wb'))
pkl.load(open(fperf_pla, 'rb'))


data/aotm-2011/setting2/perf-br2.pkl
Out[38]:
{'aotm2011': {'Test': {'AUC': 0.627090999140205,
   'HitRate': {5: 0.001587976034357113,
    10: 0.0027657937111123417,
    20: 0.004221387003560786,
    30: 0.005599440245325515,
    50: 0.007675543780819913,
    100: 0.011617975430715435,
    200: 0.018224322755395896,
    300: 0.02354979472537126,
    500: 0.03311129557353699,
    1000: 0.05200617868578454,
    3000: 0.10121437974550149,
    10000: 0.21445432546668075,
    114428: 1.0}}}}

Popularity based recommendation


In [ ]:
playlists2 = pkl.load(gzip.open(os.path.join(base_dir, 'playlists_train_dev_test_s2.pkl.gz'), 'rb'))
song2pop = pkl.load(gzip.open('data/aotm-2011/song2pop.pkl.gz', 'rb'))

In [ ]:
all_songs = pkl.load(gzip.open('data/aotm-2011/all_songs.pkl.gz', 'rb'))
index2song = {ix: sid for ix, sid in enumerate(all_songs)}

In [ ]:
song2pop_test = song2pop.copy()
for ppl in playlists2['test_playlists_held']:
    for sid in ppl:
        song2pop_test[sid] -= 1

In [ ]:
aucs_pop = []
hitrates_pop = {top: [] for top in TOPs}

assert Y_test.shape == PU_test.shape
for j in range(Y_test.shape[1]):
    if (j+1) % 10 == 0:
        sys.stdout.write('\r%d / %d' % (j+1, Y_test.shape[1]))
        sys.stdout.flush()
    y1 = Y_test[:, j].toarray().reshape(-1)
    y2 = PU_test[:, j].toarray().reshape(-1)
    indices = np.where(0 == y2)[0]
    y_true = y1[indices]
    y_pred = np.array([song2pop_test[index2song[ix]] for ix in indices])
    
    # auc
    aucs_pop.append(roc_auc_score(y_true, y_pred))
    
    # hitrates
    sortix = np.argsort(-y_pred)
    npos = np.sum(y_true)
    for top in TOPs:
        topix = sortix[:top]
        hitrates_pop[top].append(np.sum(y_true[topix]) / npos)

In [ ]:
pop_perf = {'aotm2011': {'Test': {'AUC': np.mean(aucs_pop), 
                                  'HitRate': {top: np.mean(hitrates_pop[top]) for top in hitrates_pop}}}}
pop_perf

In [ ]:
fperf_pop = os.path.join(base_dir, 'perf-pop.pkl')
print(fperf_pop)
pkl.dump(pop_perf, open(fperf_pop, 'wb'))
pkl.load(open(fperf_pop, 'rb'))

PLA


In [43]:
X = pkl.load(gzip.open(os.path.join(base_dir, 'X_train.pkl.gz'), 'rb'))
Y_train_dev = pkl.load(gzip.open(os.path.join(base_dir, 'Y_train_dev.pkl.gz'), 'rb'))
PU_dev = pkl.load(gzip.open(os.path.join(base_dir, 'PU_dev.pkl.gz'), 'rb'))

In [44]:
Y_dev = Y[:, -PU_dev.shape[1]:]
print(Y_dev.shape)
Y_dev.sum(axis=0)


(114428, 8471)
Out[44]:
matrix([[ 7,  7,  5, ..., 12, 11, 10]], dtype=int64)

In [16]:
C1 = [0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30, 100, 300, 1000]
C2 = [0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30, 100]
C3 = [0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30, 100, 300, 1000]
P = [1, 2, 3, 4, 5, 6]

offset = Y_train_dev.shape[1] - PU_dev.shape[1]
for c1 in C1:
    for c2 in C2:
        for c3 in C3:
            for p in P:
                fname = os.path.join(base_dir, 'pla-aotm2011-C-%g-%g-%g-p-%d.pkl' % (c1, c2, c3, p))
                if not os.path.exists(fname): continue
                pla = pkl.load(open(fname, 'rb'))
                W = pla.W
                b = pla.b
                print(pla)
                aucs = []
                for j in range(Y_dev.shape[1]):
                    if (j+1) % 10 == 0:
                        sys.stdout.write('\r%d / %d' % (j+1, Y_dev.shape[1]))
                        sys.stdout.flush()
                    y1 = Y_dev[:, j].toarray().reshape(-1)
                    y2 = PU_dev[:, j].toarray().reshape(-1)
                    indices = np.where(0 == y2)[0]
                    y_true = y1[indices]
                    wj = W[j + offset, :].reshape(-1)
                    y_pred = (np.dot(X, wj) + b)[indices]
                    aucs.append(roc_auc_score(y_true, y_pred))
                print(np.mean(aucs), len(aucs))


PCMLC(C1=0.03, C2=0.1, C3=0.1, p=3.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=True, weighting='both')
8470 / 84710.5029424167707581 8471
PCMLC(C1=0.03, C2=30.0, C3=3.0, p=3.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=True, weighting='both')
8470 / 84710.502459353179211 8471
PCMLC(C1=0.1, C2=0.3, C3=300.0, p=2.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=True, weighting='both')
8470 / 84710.5035719254047036 8471
PCMLC(C1=0.1, C2=30.0, C3=0.03, p=6.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=True, weighting='both')
8470 / 84710.4984555290566592 8471
PCMLC(C1=0.3, C2=0.03, C3=0.1, p=5.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=True, weighting='both')
8470 / 84710.5047973022147543 8471
PCMLC(C1=0.3, C2=30.0, C3=0.3, p=2.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=True, weighting='both')
8470 / 84710.5027027963348707 8471
PCMLC(C1=1.0, C2=0.03, C3=3.0, p=1.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=True, weighting='both')
8470 / 84710.5093990555489393 8471
PCMLC(C1=3.0, C2=10.0, C3=100.0, p=1.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=True, weighting='both')
8470 / 84710.506131690938176 8471
PCMLC(C1=3.0, C2=10.0, C3=300.0, p=1.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=True, weighting='both')
8470 / 84710.505768349081465 8471
PCMLC(C1=3.0, C2=100.0, C3=3.0, p=5.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=True, weighting='both')
8470 / 84710.5052751219255063 8471
PCMLC(C1=10.0, C2=0.03, C3=0.3, p=2.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=True, weighting='both')
8470 / 84710.5116172608315307 8471
PCMLC(C1=10.0, C2=1.0, C3=0.03, p=6.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=True, weighting='both')
8470 / 84710.5013831667033799 8471
PCMLC(C1=30.0, C2=0.01, C3=0.1, p=4.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=True, weighting='both')
8470 / 84710.5132504988791505 8471
PCMLC(C1=30.0, C2=0.1, C3=30.0, p=6.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=True, weighting='both')
8470 / 84710.5087143785703871 8471
PCMLC(C1=30.0, C2=0.3, C3=1.0, p=1.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=True, weighting='both')
8470 / 84710.5061929721876771 8471
PCMLC(C1=30.0, C2=1.0, C3=0.03, p=1.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=True, weighting='both')
8470 / 84710.49815681963656727 8471
PCMLC(C1=30.0, C2=100.0, C3=3.0, p=4.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=True, weighting='both')
8470 / 84710.5034530213853349 8471
PCMLC(C1=100.0, C2=0.1, C3=3.0, p=6.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=True, weighting='both')
8470 / 84710.5087769807733808 8471
PCMLC(C1=100.0, C2=0.3, C3=300.0, p=1.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=True, weighting='both')
8470 / 84710.5069560061507581 8471
PCMLC(C1=100.0, C2=3.0, C3=1000.0, p=2.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=True, weighting='both')
8470 / 84710.505822100331194 8471
PCMLC(C1=300.0, C2=10.0, C3=10.0, p=3.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=True, weighting='both')
8470 / 84710.5044825221131157 8471
PCMLC(C1=300.0, C2=30.0, C3=3.0, p=2.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=True, weighting='both')
8470 / 84710.5032582324318519 8471
PCMLC(C1=300.0, C2=30.0, C3=100.0, p=5.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=True, weighting='both')
8470 / 84710.5058775786221787 8471
PCMLC(C1=1000.0, C2=30.0, C3=100.0, p=3.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=True, weighting='both')
8470 / 84710.5058941273236517 8471

In [45]:
offset = Y_train_dev.shape[1] - PU_dev.shape[1]
for c1 in C1:
    for c2 in C2:
        for c3 in C3:
            for p in P:
                fname = os.path.join(base_dir, 'pla_reg/pla-aotm2011-C-%g-%g-%g-p-%d.pkl' % (c1, c2, c3, p))
                if not os.path.exists(fname): continue
                pla = pkl.load(open(fname, 'rb'))
                W = pla.W
                b = pla.b
                print(pla)
                aucs = []
                for j in range(Y_dev.shape[1]):
                    if (j+1) % 10 == 0:
                        sys.stdout.write('\r%d / %d' % (j+1, Y_dev.shape[1]))
                        sys.stdout.flush()
                    y1 = Y_dev[:, j].toarray().reshape(-1)
                    y2 = PU_dev[:, j].toarray().reshape(-1)
                    indices = np.where(0 == y2)[0]
                    y_true = y1[indices]
                    wj = W[j + offset, :].reshape(-1)
                    y_pred = (np.dot(X, wj) + b)[indices]
                    aucs.append(roc_auc_score(y_true, y_pred))
                print('\n%.5f, %d' % (np.mean(aucs), len(aucs)))


PCMLC(C1=0.1, C2=0.01, C3=1000.0, p=4.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=False, weighting='both')
8470 / 8471
0.50469, 8471
PCMLC(C1=1.0, C2=0.01, C3=1000.0, p=3.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=False, weighting='both')
8470 / 8471
0.50751, 8471
PCMLC(C1=10.0, C2=10.0, C3=1000.0, p=6.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=False, weighting='both')
8470 / 8471
0.50449, 8471
PCMLC(C1=30.0, C2=0.1, C3=30.0, p=5.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=False, weighting='both')
8470 / 8471
0.50867, 8471
PCMLC(C1=30.0, C2=300.0, C3=1000.0, p=3.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=False, weighting='both')
8470 / 8471
0.50398, 8471
PCMLC(C1=1000.0, C2=10.0, C3=100.0, p=2.0,
   similarMat=<84710x84710 sparse matrix of type '<class 'numpy.bool_'>'
	with 5111348 stored elements in Compressed Sparse Row format>,
   userwiseReg=False, weighting='both')
6650 / 8471
---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-45-bda2a9f6c359> in <module>()
     21                     wj = W[j + offset, :].reshape(-1)
     22                     y_pred = (np.dot(X, wj) + b)[indices]
---> 23                     aucs.append(roc_auc_score(y_true, y_pred))
     24                 print('\n%.5f, %d' % (np.mean(aucs), len(aucs)))

~/apps/miniconda3/lib/python3.6/site-packages/sklearn/metrics/ranking.py in roc_auc_score(y_true, y_score, average, sample_weight)
    275     return _average_binary_score(
    276         _binary_roc_auc_score, y_true, y_score, average,
--> 277         sample_weight=sample_weight)
    278 
    279 

~/apps/miniconda3/lib/python3.6/site-packages/sklearn/metrics/base.py in _average_binary_score(binary_metric, y_true, y_score, average, sample_weight)
     73 
     74     if y_type == "binary":
---> 75         return binary_metric(y_true, y_score, sample_weight=sample_weight)
     76 
     77     check_consistent_length(y_true, y_score, sample_weight)

~/apps/miniconda3/lib/python3.6/site-packages/sklearn/metrics/ranking.py in _binary_roc_auc_score(y_true, y_score, sample_weight)
    270 
    271         fpr, tpr, tresholds = roc_curve(y_true, y_score,
--> 272                                         sample_weight=sample_weight)
    273         return auc(fpr, tpr, reorder=True)
    274 

~/apps/miniconda3/lib/python3.6/site-packages/sklearn/metrics/ranking.py in roc_curve(y_true, y_score, pos_label, sample_weight, drop_intermediate)
    532     """
    533     fps, tps, thresholds = _binary_clf_curve(
--> 534         y_true, y_score, pos_label=pos_label, sample_weight=sample_weight)
    535 
    536     # Attempt to drop thresholds corresponding to points in between and

~/apps/miniconda3/lib/python3.6/site-packages/sklearn/metrics/ranking.py in _binary_clf_curve(y_true, y_score, pos_label, sample_weight)
    343 
    344     # sort scores and corresponding truth values
--> 345     desc_score_indices = np.argsort(y_score, kind="mergesort")[::-1]
    346     y_score = y_score[desc_score_indices]
    347     y_true = y_true[desc_score_indices]

~/apps/miniconda3/lib/python3.6/site-packages/numpy/core/fromnumeric.py in argsort(a, axis, kind, order)
    938 
    939     """
--> 940     return _wrapfunc(a, 'argsort', axis=axis, kind=kind, order=order)
    941 
    942 

~/apps/miniconda3/lib/python3.6/site-packages/numpy/core/fromnumeric.py in _wrapfunc(obj, method, *args, **kwds)
     50 def _wrapfunc(obj, method, *args, **kwds):
     51     try:
---> 52         return getattr(obj, method)(*args, **kwds)
     53 
     54     # An AttributeError occurs if the object does not have

KeyboardInterrupt: 

Matrix Factorisation


In [ ]:
base_dir = 'data/aotm-2011/setting2'
Y_train_dev = pkl.load(gzip.open(os.path.join(base_dir, 'Y_train_dev.pkl.gz'), 'rb'))
PU_test = pkl.load(gzip.open(os.path.join(base_dir, 'PU_test.pkl.gz'), 'rb'))

In [ ]:
ftrain1 = os.path.join(base_dir, 'mf-train_p1.csv')
ftest  = os.path.join(base_dir, 'mf-test.csv')

In [ ]:
Y_train_dev.shape

In [ ]:
PU_test.sum()

In [ ]:
lines = []

for i in range(PU_test.shape[0]):
    if (i+1) % 1000 == 0:
        sys.stdout.write('\r%d / %d' % (i+1, PU_test.shape[0])); sys.stdout.flush()
    nzcol = PU_test[i, :].nonzero()[0]
    lines += [','.join([ustrs[i], istrs2[j], '5\n']) for j in nzcol]

In [ ]:
len(lines)

In [ ]:
lines[0]

In [ ]:
with open(ftrain1, 'w') as fd:
    fd.writelines(lines)

In [ ]:
ustrs = ['U%d' % i for i in range(Y_train_dev.shape[0])]
istrs1 = ['P%d' % j for j in range(Y_train_dev.shape[1])]

In [ ]:
lines = []

for i in range(Y_train_dev.shape[0]):
    if (i+1) % 10 == 0:
        sys.stdout.write('\r%d / %d' % (i+1, Y_train_dev.shape[0])); sys.stdout.flush()
    # convert True to rating 5 and False to 1
    lines += [','.join([ustrs[i], istrs1[j], '5\n']) if Y_train_dev[i, j] is True else \
              ','.join([ustrs[i], istrs1[j], '1\n']) for j in range(Y_train_dev.shape[1])]

In [ ]:
with open(ftrain, 'w') as fd:
    for i in range(Y_train_dev.shape[0]):
        sys.stdout.write('\r%d / %d' % (i+1, Y_train_dev.shape[0])); sys.stdout.flush()
        istr = 'U%d' % i
        for j in range(Y_train_dev.shape[1]):
            #print(Y_train[i, j])
            jstr = 'P%d' % j
            v = Y_train_dev[i, j]
            rating = 1   # convert False to rating 1
            if v is True:
                rating = 5   # convert True to rating 5
            line = '%s,%s,%d\n' % (istr, jstr, rating)
            #print(line)
            fd.write(line)

Evaluate on dev set --BR


In [ ]:
rows = ['LR', 'PC', 'LR-2017']
cols = ['F1', 'Precision@K']
df = pd.DataFrame(index=rows, columns=cols)
df.head()

In [ ]:
C = 1
#fname = os.path.join('data', 'aotm2011-params-br/br-aotm2011-C-%s.pkl' % str(C))
fname = os.path.join(data_dir, 'br-aotm2011-C-%g.pkl' % C)
br = pkl.load(open(fname, 'rb'))

Evaluate F1: threshold for logistic regression is 0 for logits, 0.5 for probabilities.


In [ ]:
F1 = evaluate_minibatch(br, calc_F1, X_dev, Y_dev, threshold=0, batch_size=1500, verbose=1)
avgF1 = np.mean(F1)
F1_all.append(avgF1)
print('\nF1: %g' % avgF1)

In [ ]:
np.mean(F1)

C: 0.1, Threshold: 0.05, F1: 0.00254648
C: 1, Threshold: 0.05, F1: 0.0121401

Evaluate Precision@K.


In [ ]:
pak = evaluate_minibatch(br, calc_precisionK, X_dev, Y_dev, threshold=None, batch_size=1500, verbose=1)
print('\nPrecision@K: %g' % np.mean(pak))

C: 0.1, Precision@K: 0.0884917
C: 1, Precision@K: 0.0943461

Evaluate on test set -- BR


In [ ]:
best_C = 1
best_TH = 0.05
fname = os.path.join('data', 'aotm2011-params-br/br-aotm2011-C-%s.pkl' % str(best_C))
best_br = pkl.load(open(fname, 'rb'))

In [ ]:
F1_test_br = evaluate_minibatch(best_br, calc_F1, X_test, Y_test, threshold=best_TH, batch_size=1500, verbose=1)
print('\nTest F1: %g' % np.mean(F1_test_br))

In [ ]:
pak_test_br = evaluate_minibatch(best_br, calc_precisionK, X_test, Y_test,threshold=None,batch_size=1500,verbose=1)
print('\nTest Precision@K: %g' % np.mean(pak_test_br))

In [ ]:
df.loc['LR', 'F1'] = np.mean(F1_test_br)
df.loc['LR', 'Precision@K'] = np.mean(pak_test_br)

In [ ]:
df

Evaluate on dev set -- PC


In [ ]:
C_set = [0.1, 0.3, 1, 3, 10, 30, 100, 300, 1000, 3000, 10000, 30000]
p_set = [1, 2, 3, 4, 5, 6, 7, 8]
metrics_pc = [ ]
print('%15s %15s %15s %15s %15s' % ('C', 'p', 'Threshold', 'F1', 'Precision@K'))
for C in C_set:
    for p in p_set:
        #fname = os.path.join('data', 'aotm2011-params-pc/pc-aotm2011-C-%g-p-%g.pkl' % (C, p))
        fname = os.path.join(data_dir, 'pc-aotm2011-C-%g-p-%g.pkl' % (C, p))
        if not os.path.exists(fname): continue
        pc_dict = pkl.load(open(fname, 'rb'))
        print('%15s %15s %15s %15s %15s' % ('%g'%pc_dict['C'], '%g'%pc_dict['p'], \
                                            '%g'%pc_dict['Threshold'], '%g'%pc_dict['F1'], \
                                            '%g'%pc_dict['Precision@K']))
        metrics_pc.append((pc_dict['C'], pc_dict['p'], pc_dict['Threshold'],pc_dict['F1'],pc_dict['Precision@K']))
        clf = PClassificationMLC()
        clf.load_params(fname)
        th = pc_dict['Threshold']
        F1 = evaluate_minibatch(clf, calc_F1, X_test, Y_test, threshold=th, batch_size=1500, verbose=1)
        print('\nTest F1: %g' % np.mean(F1))
        pak = evaluate_minibatch(clf, calc_precisionK, X_test, Y_test, threshold=None, batch_size=1500, verbose=1)
        print('\nTest Precision@K: %g' % np.mean(pak))

In [ ]:
keyix = 3  # F1
sorted_metrics_pc = sorted(metrics_pc, key=lambda x: x[keyix], reverse=True)
print('Best hyper-param:\n(C, p, Threshold, F1, Precision@K):', sorted_metrics_pc[0])

Evaluate on test set -- PC


In [ ]:
best_C = 30000 #10000 #300   #3000
best_p = 2 #2 #3     #6
best_TH = 0.1 #0.15 #0.1
#fname = os.path.join('data', 'aotm2011-params-pc/pc-aotm2011-C-%g-p-%g.pkl' % (best_C, best_p))
fname = os.path.join(data_dir, 'pc-aotm2011-C-%g-p-%g.pkl' % (best_C, best_p))
best_pc = PClassificationMLC()
best_pc.load_params(fname)

In [ ]:
bestdict = pkl.load(open(fname, 'rb'))

In [ ]:
plt.plot(bestdict['cost'])

In [ ]:
F1_test_pc = evaluate_minibatch(best_pc, calc_F1, X_test, Y_test, threshold=best_TH, batch_size=1500, verbose=1)
print('\nTest F1: %g' % np.mean(F1_test_pc))

In [ ]:
pak = evaluate_minibatch(best_pc, calc_precisionK, X_train, Y_train, threshold=None, batch_size=1500, verbose=1)

In [ ]:
print('\nTrain P@K: %g' % np.mean(pak))

In [ ]:
preds = best_pc.decision_function(X_train[:10])

In [ ]:
test_ex_idx = 2

plt.hist(preds[test_ex_idx], bins=50)

y_true = Y_train[test_ex_idx].toarray()

pos_idx = np.where(y_true)[1]
print('prediction of true positives')
print(preds[test_ex_idx][pos_idx])
print('top predictions')
np.sort(preds[test_ex_idx])[-20:]

In [ ]:
pak_test_pc = evaluate_minibatch(best_pc, calc_precisionK, X_test, Y_test,threshold=None,batch_size=1500,verbose=1)
print('\nTest Precision@K: %g' % np.mean(pak_test_pc))

In [ ]:
df.loc['PC', 'F1'] = np.mean(F1_test_pc)
df.loc['PC', 'Precision@K'] = np.mean(pak_test_pc)

Result table


In [ ]:
df.loc['LR-2017', 'F1'] = 0.031

In [ ]:
tab_str = df.to_latex(float_format=lambda x: '$%.4f$' % x, na_rep='-', multirow=False, escape=False)

In [ ]:
print('\\begin{table}[!h]')
print('\centering')
print('\\caption{Performance on test set}')
print('\\label{tab:perf}')    
print(tab_str)
print('\\end{table}')