In [1]:
%matplotlib inline
%load_ext autoreload
%autoreload 2
import os, sys, time, gzip
import pickle as pkl
import numpy as np
import pandas as pd
from sklearn.metrics import precision_recall_fscore_support, roc_auc_score, average_precision_score
from scipy.sparse import lil_matrix, issparse
import matplotlib.pyplot as plt
import seaborn as sns
In [2]:
sys.path.append('src')
from PCMLC import PCMLC as PClassificationMLC
from PCMLC import obj_pclassification
from BinaryRelevance import BinaryRelevance
from evaluate import f1_score_nowarn, calc_F1, calc_precisionK, evaluate_minibatch, evalPred
In [3]:
data_dir = 'data/aotm-2011/setting1'
fxtrain = os.path.join(data_dir, 'X_train.pkl.gz')
fytrain = os.path.join(data_dir, 'Y_train.pkl.gz')
fxdev = os.path.join(data_dir, 'X_dev.pkl.gz')
fydev = os.path.join(data_dir, 'Y_dev.pkl.gz')
fxtest = os.path.join(data_dir, 'X_test.pkl.gz')
fytest = os.path.join(data_dir, 'Y_test.pkl.gz')
In [26]:
TOPs = [5, 10, 20, 30, 50, 100, 200, 300, 500, 1000]
In [5]:
X_train = pkl.load(gzip.open(fxtrain, 'rb'))
Y_train = pkl.load(gzip.open(fytrain, 'rb'))
X_dev = pkl.load(gzip.open(fxdev, 'rb'))
Y_dev = pkl.load(gzip.open(fydev, 'rb'))
X_test = pkl.load(gzip.open(fxtest, 'rb'))
Y_test = pkl.load(gzip.open(fytest, 'rb'))
In [6]:
print('Train: %15s %15s' % (X_train.shape, Y_train.shape))
print('Dev : %15s %15s' % (X_dev.shape, Y_dev.shape))
print('Test : %15s %15s' % (X_test.shape, Y_test.shape))
In [ ]:
def calc_auc_hitrate(estimator, X, Y, top=100, useLoop=False):
N, D = X.shape
K = Y.shape[1]
assert Y.shape[0] == N
assert top <= N
if issparse(Y):
#Y = Y.tocsc().astype(np.bool)
npos = np.array(Y.sum(axis=0))[0] # be careful with sparse matrix
else:
npos = np.sum(Y, axis=0)
nzcol = np.nonzero(npos)[0] # need columns with at least one True
if useLoop is False:
Y_pred = estimator.predict(X)
auc = roc_auc_score(Y[:, nzcol], Y_pred[:, nzcol], average='macro')
topix = np.argsort(-Y_pred, axis=0)[:top, :]
hitrate = np.mean([np.sum(Y[topix[:, col], col]) / npos[col] for col in nzcol])
return auc, hitrate
else:
W = estimator.W
b = estimator.b
assert W.shape == (K, D)
aucs = []
hitrates = []
for col in nzcol:
if (col+1) % 100 == 0:
sys.stdout.write('\r{:,} / {:,}'.format(col+1, nzcol[-1]))
sys.stdout.flush()
pred = np.dot(X, W[col, :]) + b
#aucs.append(roc_auc_score(Y[:, col], pred)) # TypeError: len() of unsized object
y_true = Y[:, col].toarray()
aucs.append(roc_auc_score(y_true, pred))
topix = np.argsort(-pred)[:top]
hitrates.append(np.sum(Y[topix, col]) / npos[col])
return np.mean(aucs), np.mean(hitrates)
In [ ]:
def summary_auc(results):
assert type(results) == list
aucs = np.array([t[0] for t in results])
nums = np.array([t[1] for t in results])
return np.sum(aucs * nums) / np.sum(nums)
def calc_hitrate_pl(Y_true, Y_pred, top=100):
assert Y_true.shape == Y_pred.shape
assert top <= Y_true.shape[0]
nzcol = np.nonzero(np.sum(Y_true, axis=0))[0] # columns with at least one True
topix = np.argsort(-Y_pred, axis=0)[:top, :]
npos = np.sum(Y_true, axis=0)
return np.array([np.sum(Y_true[topix[:, col], col]) / npos[col] for col in nzcol])
In [ ]:
def eval_pl(Y_true, Y_pred, top=100, useLoop=False):
if useLoop is False:
assert Y_true.shape == Y_pred.shape
assert top <= Y_true.shape[0]
nzcol = np.nonzero(np.sum(Y_true, axis=0))[0] # columns with at least one True
ncols = len(nzcol)
topix = np.argsort(-Y_pred, axis=0)[:top, :]
npos = np.sum(Y_true, axis=0)
hr = np.mean([np.sum(Y_true[topix[:, j], j]) / npos[j] for j in nzcol])
paks, valid_indices = calc_precisionK(Y_true.T, Y_pred.T)
pak = np.mean(paks[valid_indices])
auc = roc_auc_score(Y_true[:, nzcol], Y_pred[:, nzcol], average='macro')
ap = average_precision_score(Y_true[:, nzcol], Y_pred[:, nzcol], average='macro')
nrr = mean_normalised_reciprocal_rank(Y_true, Y_pred)
else:
assert type(Y_true) == list
assert type(Y_pred) == list
assert len(Y_true) == len(Y_pred)
hitrates, paks, aucs, aps, nrrs = [], [], [], [], []
for j in range(len(Y_true)):
if np.sum(Y_true[j]) < 1: continue # filtering out cases where all ground truths are negative.
gt = Y_true[j].reshape(-1)
pred = Y_pred[j].reshape(-1)
assert gt.shape == pred.shape
assert top <= gt.shape[0]
topix = np.argsort(-pred)[:top]
hitrates.append(np.sum(gt[topix]) / np.sum(gt))
#paks.append(calc_precisionK(gt.reshape(1,-1), pred.reshape(1,-1))) # incorrect
paks.append(evalPred(gt, pred, metricType='Precision@K'))
aucs.append(roc_auc_score(gt, pred))
aps.append(average_precision_score(gt, pred))
nrrs.append(mean_normalised_reciprocal_rank(gt.reshape(-1,1), pred.reshape(-1,1)))
hr, pak, auc, ap, nrr = [np.mean(x) for x in [hitrates, paks, aucs, aps, nrrs]]
ncols = len(paks)
print('Average over %d columns' % ncols)
print('%-20s %.4f' % ('Mean HitRate@100:', hr))
print('%-20s %.4f' % ('Mean P@K:', pak))
print('%-20s %.4f' % ('Mean AUC:', auc))
print('%-20s %.4f' % ('MAP:', ap))
print('%-20s %.4f' % ('Mean NRR:', nrr))
In [ ]:
def calc_auc_pl(Y_true, Y_pred, useLoop=False):
if useLoop is False:
assert Y_true.shape == Y_pred.shape
npos = np.asarray(np.sum(Y_true, axis=0)).reshape(-1) # 1D array, works for both sparse and dense matrix
nzcol = np.nonzero(npos)[0] # columns with at least one True
Y_part = Y_true[:, nzcol]
if issparse(Y_part):
Y_part = Y_part.toarray()
auc = roc_auc_score(Y_part, Y_pred[:, nzcol], average='macro')
return (auc, len(nzcol))
else:
assert type(Y_true) == list
assert type(Y_pred) == list
assert len(Y_true) == len(Y_pred)
aucs = []
for j in range(len(Y_true)):
if np.sum(Y_true[j]) < 1: continue # filtering out cases where all ground truths are negative.
gt = Y_true[j].reshape(-1)
pred = Y_pred[j].reshape(-1)
assert gt.shape == pred.shape
aucs.append(roc_auc_score(gt, pred))
return (np.mean(aucs), len(aucs))
In [ ]:
def calc_hitrate_pl(Y_true, Y_pred, tops=[100], useLoop=False):
"""
Compute hitrate at top-N.
"""
if useLoop is False:
assert Y_true.shape == Y_pred.shape
assert type(tops) == list
hitrates = []
sortix = np.argsort(-Y_pred, axis=0)
npos = np.asarray(np.sum(Y_true, axis=0)).reshape(-1) # 1D array, works for both sparse and dense matrix
nzcol = np.nonzero(npos)[0] # columns with at least one True
for top in tops:
assert 0 < top <= Y_true.shape[0]
topix = sortix[:top, :]
hr = np.mean([np.sum(Y_true[topix[:, col], col]) / npos[col] for col in nzcol])
hitrates.append(hr)
return (hitrates, len(nzcol))
else:
assert type(Y_true) == list
assert type(Y_pred) == list
assert len(Y_true) == len(Y_pred)
assert type(tops) == list
top_dict = {top: [] for top in tops}
for j in range(len(Y_true)):
if np.sum(Y_true[j]) < 1: continue # filtering out cases where all ground truths are negative.
gt = Y_true[j].reshape(-1)
pred = Y_pred[j].reshape(-1)
sortix = np.argsort(-pred)
assert gt.shape == pred.shape
for top in tops:
assert 0 < top <= gt.shape[0]
topix = sortix[:top]
top_dict[top].append(np.sum(gt[topix]) / np.sum(gt))
ncols = len(top_dict[tops[0]])
for top in tops:
assert ncols == len(top_dict[top])
return ([np.mean(top_dict[top]) for top in tops], ncols)
In [7]:
C1 = [0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30, 100, 300, 1000, 3000]
C2 = [0.01, 0.1, 0.2, 0.4, 0.6, 0.8, 1, 3, 10, 30, 100, 300, 1000]
C3 = [0.01, 0.1, 0.3, 1, 2, 4, 6, 8, 10, 20, 30, 100, 300, 1000]
P = [1, 2, 3, 4, 5, 6]
In [ ]:
%%script false
for c1 in C1:
for c2 in C2:
for c3 in C3:
for p in P:
fname = os.path.join(data_dir, 'mlr-aotm2011-C-%g-%g-%g-p-%g.pkl' % (c1, c2, c3, p))
if os.path.exists(fname):
clf = pkl.load(open(fname, 'rb'))
print(clf)
auc_dev, hr_dev = calc_auc_hitrate(clf, X_dev, Y_dev, useLoop=True)
print('{:10s}: AUC = {:.4f}, HitRate@100 = {:.4f}'.format('Dev set', auc_dev, hr_dev))
auc_test, hr_test = calc_auc_hitrate(clf, X_test, Y_test, useLoop=True)
print('{:10s}: AUC = {:.4f}, HitRate@100 = {:.4f}'.format('Test set', auc_test, hr_test))
print('-'*50, '\n')
In [8]:
def calc_auc_w(W, b, X, Y):
aucs = []
for j in range(Y.shape[1]):
if (j+1) % 100 == 0:
sys.stdout.write('\r%d / %d' % (j+1, Y.shape[1]))
sys.stdout.flush()
y_true = Y[:, j]
if issparse(Y):
y_true = y_true.toarray().reshape(-1)
if np.sum(y_true) < 1: continue
y_pred = np.dot(X, W[j, :]) + b
aucs.append(roc_auc_score(y_true, y_pred))
return aucs
In [ ]:
def calc_hitrate_w(W, b, X, Y, tops=[100]):
"""
Compute hitrate at top-N.
"""
top_dict = {top: [] for top in tops}
for j in range(Y.shape[1]):
if (j+1) % 100 == 0:
sys.stdout.write('\r%d / %d' % (j+1, Y.shape[1]))
sys.stdout.flush()
y_true = Y[:, j]
if issparse(Y):
y_true = y_true.toarray().reshape(-1)
npos = np.sum(y_true)
if npos < 1: continue
y_pred = np.dot(X, W[j, :]) + b
y_pred = y_pred.reshape(-1)
sortix = np.argsort(-y_pred)
assert y_true.shape == y_pred.shape
for top in tops:
assert 0 < top <= y_true.shape[0]
topix = sortix[:top]
top_dict[top].append(np.sum(y_true[topix]) / npos)
ncols = len(top_dict[tops[0]])
for top in tops:
assert ncols == len(top_dict[top])
return ({top: np.mean(top_dict[top]) for top in tops}, ncols)
In [28]:
w = np.load('data/aotm-2011/setting1/mlr_lineareg/both-1000-0.01-30-5-latest.npy')
aucs1 = calc_auc_w(W=w[1:].reshape(K, D), b=w[0], X=X_dev, Y=Y_dev)
print('\n%.4f, %d / %d' % (np.mean(aucs1), len(aucs1), Y_dev.shape[1]))
In [29]:
w = np.load('data/aotm-2011/setting1/mlr_lineareg/both-1000-0.01-30-5-latest.npy')
aucs2 = calc_auc_w(W=w[1:].reshape(K, D), b=w[0], X=X_test, Y=Y_test)
print('\n%.4f, %d / %d' % (np.mean(aucs2), len(aucs2), Y_test.shape[1]))
In [32]:
for c1 in C1:
if c1 == 0.03: continue
for p in P:
fname = os.path.join(data_dir, 'mlr_samples/mlr-aotm2011-samples-C-%g-1-1-p-%d.pkl' % (c1, p))
if not os.path.exists(fname):continue
print(fname)
mlr = pkl.load(open(fname, 'rb'))
aucs_mlr = calc_auc_w(W=mlr.W, b=mlr.b, X=X_test, Y=Y_test)
print('\n%.5f, %d / %d' % (np.mean(aucs_mlr), len(aucs_mlr), Y_test.shape[1]))
In [11]:
D = X_train.shape[1]
K = Y_train.shape[1]
for c1 in C1:
for p in P:
fname = os.path.join(data_dir, 'mlr_samples/mlr-N-samples-%g-1-1-%d-latest.npy' % (c1, p))
if not os.path.exists(fname):continue
print(fname)
w = np.load(fname)
aucs = calc_auc_w(W=w[1:].reshape(K, D), b=w[0], X=X_dev, Y=Y_dev)
print('\n%.5f, %d / %d\n' % (np.mean(aucs), len(aucs), Y_dev.shape[1]))
In [13]:
print('Choose: mlr_samples/mlr-N-samples-10-1-1-3-latest.npy')
In [13]:
print('Choose: mlr_samples/mlr-N-samples-30-1-1-1-latest.npy')
In [14]:
D = X_train.shape[1]
K = Y_train.shape[1]
for c1 in C1:
for p in P:
fname = os.path.join(data_dir, 'mlr_labels/mlr-N-labels-%g-1-1-%d-latest.npy' % (c1, p))
if not os.path.exists(fname):continue
print(fname)
w = np.load(fname)
aucs = calc_auc_w(W=w[1:].reshape(K, D), b=w[0], X=X_dev, Y=Y_dev)
print('\n%.5f, %d / %d\n' % (np.mean(aucs), len(aucs), Y_dev.shape[1]))
In [15]:
print('Choose:', 'data/aotm-2011/setting1/mlr_labels/mlr-N-labels-30-1-1-3-latest.npy')
In [16]:
D = X_train.shape[1]
K = Y_train.shape[1]
for c1 in C1:
for c2 in C2:
for p in P:
fname = os.path.join(data_dir, 'mlr_both/mlr-N-both-%g-%g-1-%d-latest.npy' % (c1, c2, p))
if not os.path.exists(fname):continue
print(fname)
w = np.load(fname)
aucs = calc_auc_w(W=w[1:].reshape(K, D), b=w[0], X=X_dev, Y=Y_dev)
print('\n%.5f, %d / %d\n' % (np.mean(aucs), len(aucs), Y_dev.shape[1]))
In [17]:
D = X_train.shape[1]
K = Y_train.shape[1]
for c1 in C1:
for c3 in C3:
for p in P:
fname = os.path.join(data_dir, 'mlr_multitask/mlr-Y-samples-%g-1-%g-%d-latest.npy' % (c1, c3, p))
if not os.path.exists(fname):continue
print(fname)
w = np.load(fname)
aucs = calc_auc_w(W=w[1:].reshape(K, D), b=w[0], X=X_dev, Y=Y_dev)
print('\n%.5f, %d / %d\n' % (np.mean(aucs), len(aucs), Y_dev.shape[1]))
In [23]:
print('Choose:', 'data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-100-1-0.01-3-latest.npy')
In [24]:
print('Choose:', 'data/aotm-2011/setting1/mlr_multitask/mlr-Y-samples-300-1-0.01-3-latest.npy')
In [27]:
perf_mlr = {'aotm2011': {'Test': dict()}}
fperf_mlr = os.path.join(data_dir, 'perf_mlr.pkl')
In [28]:
for fname in ['mlr-N-samples-10-1-1-3-latest.npy', 'mlr-N-samples-30-1-1-1-latest.npy']:
fname = os.path.join(data_dir, 'mlr_train_dev/' + fname)
print(fname)
w = np.load(fname)
aucs = calc_auc_w(W=w[1:].reshape(K, D), b=w[0], X=X_test, Y=Y_test)
print('\n%.5f, %d / %d\n' % (np.mean(aucs), len(aucs), Y_test.shape[1]))
mean_hitrates, ncols = calc_hitrate_w(W=w[1:].reshape(K, D), b=w[0], X=X_test, Y=Y_test, tops=TOPs)
#print(mean_hitrates, ncols)
break
perf_mlr['aotm2011']['Test']['samples'] = {'AUC': np.mean(aucs), 'HitRate': mean_hitrates}
In [30]:
perf_mlr
Out[30]:
In [ ]:
fname = os.path.join(data_dir, 'mlr_train_dev/mlr-N-labels-30-1-1-3-latest.npy')
print(fname)
w = np.load(fname)
aucs = calc_auc_w(W=w[1:].reshape(K, D), b=w[0], X=X_test, Y=Y_test)
print('\n%.5f, %d / %d\n' % (np.mean(aucs), len(aucs), Y_test.shape[1]))
mean_hitrates, ncols = calc_hitrate_w(W=w[1:].reshape(K, D), b=w[0], X=X_test, Y=Y_test, tops=TOPs)
#print(mean_hitrates, ncols)
perf_mlr['aotm2011']['Test']['labels'] = {'AUC': np.mean(aucs), 'HitRate': mean_hitrates}
In [ ]:
fname = os.path.join(data_dir, 'mlr_train_dev/mlr-N-both-3-0.01-1-3-latest.npy')
print(fname)
w = np.load(fname)
aucs = calc_auc_w(W=w[1:].reshape(K, D), b=w[0], X=X_test, Y=Y_test)
print('\n%.5f, %d / %d\n' % (np.mean(aucs), len(aucs), Y_test.shape[1]))
mean_hitrates, ncols = calc_hitrate_w(W=w[1:].reshape(K, D), b=w[0], X=X_test, Y=Y_test, tops=TOPs)
#print(mean_hitrates, ncols)
perf_mlr['aotm2011']['Test']['both'] = {'AUC': np.mean(aucs), 'HitRate': mean_hitrates}
In [ ]:
fname = os.path.join(data_dir, '')
print(fname)
w = np.load(fname)
aucs = calc_auc_w(W=w[1:].reshape(K, D), b=w[0], X=X_test, Y=Y_test)
print('\n%.5f, %d / %d\n' % (np.mean(aucs), len(aucs), Y_test.shape[1]))
mean_hitrates, ncols = calc_hitrate_w(W=w[1:].reshape(K, D), b=w[0], X=X_test, Y=Y_test, tops=TOPs)
#print(mean_hitrates, ncols)
perf_mlr['aotm2011']['Test']['multitask'] = {'AUC': np.mean(aucs), 'HitRate': mean_hitrates}
In [35]:
base_dir = 'data/aotm-2011/setting1'
br_name = 'br1'
br_dir = os.path.join(base_dir, br_name)
fsplit = os.path.join(br_dir, br_name + '.split')
In [ ]:
X_test = pkl.load(gzip.open(os.path.join(base_dir, 'X_test.pkl.gz'), 'rb'))
Y_test = pkl.load(gzip.open(os.path.join(base_dir, 'Y_test.pkl.gz'), 'rb'))
In [ ]:
aucs = []
with open(fsplit, 'r') as fd:
for line in fd:
start, end = line.strip().split(' ')
print(start, end)
fname = os.path.join(br_dir, 'br1-aotm2011-%s-%s.pkl' % (start, end))
#print(fname)
#break
br = pkl.load(open(fname, 'rb'))
pred = br.predict(X_test)
Y_part = Y_test[:, int(start):int(end)]
nzcol = np.nonzero(np.sum(Y_part, axis=0))[1]
#print(np.sum(Y_part, axis=0))
#print('-'*20)
#print(nzcol)
auc, npl = calc_auc_pl(Y_part[:, nzcol].toarray(), pred[:, nzcol])
aucs.append((auc, npl))
In [ ]:
mean_auc = np.sum([np.prod(t) for t in aucs]) / np.sum([t[1] for t in aucs])
print(mean_auc)
Hit rate curve.
In [ ]:
Y_test.shape
In [ ]:
hitrates = [] # [(hitrates, ncols)]
with open(fsplit, 'r') as fd:
for line in fd:
start, end = line.strip().split(' ')
print(start, end)
fname = os.path.join(br_dir, 'br1-aotm2011-%s-%s.pkl' % (start, end))
br = pkl.load(open(fname, 'rb'))
pred = br.predict(X_test)
Y_part = Y_test[:, int(start):int(end)]
hitrates.append(calc_hitrate_pl(Y_part, pred, tops=TOPs))
In [ ]:
mean_hitrate_dict = dict()
denom = np.sum([t[1] for t in hitrates])
for i in range(len(tops)):
mean_hitrate_dict[tops[i]] = np.sum([t[0][i] * t[1] for t in hitrates]) / denom
#print(mean_hitrate_dict)
In [ ]:
xx = sorted(mean_hitrate_dict.keys())
yy = [mean_hitrate_dict[top] for top in xx]
ax = plt.subplot(111)
ax.plot(xx, yy, ls='--', marker='o')
ax.set_xlabel('Top-N')
ax.set_ylabel('Hit Rate')
ax.set_xscale('log')
#ax.set_xlim(0, 500)
In [ ]:
br1_perf = {'aotm2011': {'Test': {'AUC': mean_auc, 'HitRate': mean_hitrate_dict}}}
br1_perf
In [36]:
fperf = os.path.join(base_dir, 'perf-' + br_name + '.pkl')
#pkl.dump(br1_perf, open(fperf, 'wb'))
pkl.load(open(fperf, 'rb'))
Out[36]:
In [37]:
base_dir = 'data/aotm-2011/setting2'
br_name = 'br2'
br_dir = os.path.join(base_dir, br_name)
fsplit = os.path.join(br_dir, br_name + '.split')
In [12]:
X = pkl.load(gzip.open(os.path.join(base_dir, 'X_train.pkl.gz'), 'rb'))
Y = pkl.load(gzip.open(os.path.join(base_dir, 'Y.pkl.gz'), 'rb'))
PU_test = pkl.load(gzip.open(os.path.join(base_dir, 'PU_test.pkl.gz'), 'rb'))
In [9]:
print(Y.shape)
print(PU_test.shape)
In [10]:
np.sum(PU_test, axis=0)
Out[10]:
In [11]:
Y_test = Y[:, -PU_test.shape[1]:]
print(Y_test.shape)
Y_test.sum(axis=0)
Out[11]:
In [ ]:
np.where(PU_test[:,0:10].toarray()[:,0] == 0)
In [ ]:
aucs_pla = []
hitrates_pla = {top: [] for top in TOPs}
with open(fsplit, 'r') as fd:
for line in fd:
start, end = line.strip().split(' ')
print(start, end)
fname = os.path.join(br_dir, br_name + '-aotm2011-%s-%s.pkl' % (start, end))
br = pkl.load(open(fname, 'rb'))
Y_pred = br.predict(X)
Y_part = Y_test[:, int(start):int(end)]
PU_part = PU_test[:, int(start):int(end)]
assert Y_part.shape == PU_part.shape == Y_pred.shape
if issparse(Y_part):
Y_part = Y_part.toarray()
if issparse(PU_part):
PU_part = PU_part.toarray()
for j in range(Y_part.shape[1]):
indices = np.where(0 == PU_part[:, j])[0]
y_true = Y_part[indices, j].reshape(-1)
y_pred = Y_pred[indices, j].reshape(-1)
#print(y_true.shape)
#print(y_pred.shape)
# auc
aucs_pla.append(roc_auc_score(y_true, y_pred))
# hitrates
sortix = np.argsort(-y_pred)
npos = np.sum(y_true)
for top in TOPs:
topix = sortix[:top]
hitrates_pla[top].append(np.sum(y_true[topix]) / npos)
In [ ]:
mean_auc_pla = np.mean(aucs_pla)
print(mean_auc_pla)
In [ ]:
#aucs_pla
In [ ]:
hitrates_pla.keys()
In [ ]:
mean_hitrates = {top: np.mean(hitrates_pla[top]) for top in hitrates_pla}
#mean_hitrates
In [ ]:
br2_perf = {'aotm2011': {'Test': {'AUC': mean_auc_pla, 'HitRate': mean_hitrates}}}
br2_perf
In [38]:
fperf_pla = os.path.join(base_dir, 'perf-' + br_name + '.pkl')
print(fperf_pla)
#pkl.dump(br2_perf, open(fperf_pla, 'wb'))
pkl.load(open(fperf_pla, 'rb'))
Out[38]:
In [ ]:
playlists2 = pkl.load(gzip.open(os.path.join(base_dir, 'playlists_train_dev_test_s2.pkl.gz'), 'rb'))
song2pop = pkl.load(gzip.open('data/aotm-2011/song2pop.pkl.gz', 'rb'))
In [ ]:
all_songs = pkl.load(gzip.open('data/aotm-2011/all_songs.pkl.gz', 'rb'))
index2song = {ix: sid for ix, sid in enumerate(all_songs)}
In [ ]:
song2pop_test = song2pop.copy()
for ppl in playlists2['test_playlists_held']:
for sid in ppl:
song2pop_test[sid] -= 1
In [ ]:
aucs_pop = []
hitrates_pop = {top: [] for top in TOPs}
assert Y_test.shape == PU_test.shape
for j in range(Y_test.shape[1]):
if (j+1) % 10 == 0:
sys.stdout.write('\r%d / %d' % (j+1, Y_test.shape[1]))
sys.stdout.flush()
y1 = Y_test[:, j].toarray().reshape(-1)
y2 = PU_test[:, j].toarray().reshape(-1)
indices = np.where(0 == y2)[0]
y_true = y1[indices]
y_pred = np.array([song2pop_test[index2song[ix]] for ix in indices])
# auc
aucs_pop.append(roc_auc_score(y_true, y_pred))
# hitrates
sortix = np.argsort(-y_pred)
npos = np.sum(y_true)
for top in TOPs:
topix = sortix[:top]
hitrates_pop[top].append(np.sum(y_true[topix]) / npos)
In [ ]:
pop_perf = {'aotm2011': {'Test': {'AUC': np.mean(aucs_pop),
'HitRate': {top: np.mean(hitrates_pop[top]) for top in hitrates_pop}}}}
pop_perf
In [ ]:
fperf_pop = os.path.join(base_dir, 'perf-pop.pkl')
print(fperf_pop)
pkl.dump(pop_perf, open(fperf_pop, 'wb'))
pkl.load(open(fperf_pop, 'rb'))
In [43]:
X = pkl.load(gzip.open(os.path.join(base_dir, 'X_train.pkl.gz'), 'rb'))
Y_train_dev = pkl.load(gzip.open(os.path.join(base_dir, 'Y_train_dev.pkl.gz'), 'rb'))
PU_dev = pkl.load(gzip.open(os.path.join(base_dir, 'PU_dev.pkl.gz'), 'rb'))
In [44]:
Y_dev = Y[:, -PU_dev.shape[1]:]
print(Y_dev.shape)
Y_dev.sum(axis=0)
Out[44]:
In [16]:
C1 = [0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30, 100, 300, 1000]
C2 = [0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30, 100]
C3 = [0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30, 100, 300, 1000]
P = [1, 2, 3, 4, 5, 6]
offset = Y_train_dev.shape[1] - PU_dev.shape[1]
for c1 in C1:
for c2 in C2:
for c3 in C3:
for p in P:
fname = os.path.join(base_dir, 'pla-aotm2011-C-%g-%g-%g-p-%d.pkl' % (c1, c2, c3, p))
if not os.path.exists(fname): continue
pla = pkl.load(open(fname, 'rb'))
W = pla.W
b = pla.b
print(pla)
aucs = []
for j in range(Y_dev.shape[1]):
if (j+1) % 10 == 0:
sys.stdout.write('\r%d / %d' % (j+1, Y_dev.shape[1]))
sys.stdout.flush()
y1 = Y_dev[:, j].toarray().reshape(-1)
y2 = PU_dev[:, j].toarray().reshape(-1)
indices = np.where(0 == y2)[0]
y_true = y1[indices]
wj = W[j + offset, :].reshape(-1)
y_pred = (np.dot(X, wj) + b)[indices]
aucs.append(roc_auc_score(y_true, y_pred))
print(np.mean(aucs), len(aucs))
In [45]:
offset = Y_train_dev.shape[1] - PU_dev.shape[1]
for c1 in C1:
for c2 in C2:
for c3 in C3:
for p in P:
fname = os.path.join(base_dir, 'pla_reg/pla-aotm2011-C-%g-%g-%g-p-%d.pkl' % (c1, c2, c3, p))
if not os.path.exists(fname): continue
pla = pkl.load(open(fname, 'rb'))
W = pla.W
b = pla.b
print(pla)
aucs = []
for j in range(Y_dev.shape[1]):
if (j+1) % 10 == 0:
sys.stdout.write('\r%d / %d' % (j+1, Y_dev.shape[1]))
sys.stdout.flush()
y1 = Y_dev[:, j].toarray().reshape(-1)
y2 = PU_dev[:, j].toarray().reshape(-1)
indices = np.where(0 == y2)[0]
y_true = y1[indices]
wj = W[j + offset, :].reshape(-1)
y_pred = (np.dot(X, wj) + b)[indices]
aucs.append(roc_auc_score(y_true, y_pred))
print('\n%.5f, %d' % (np.mean(aucs), len(aucs)))
In [ ]:
base_dir = 'data/aotm-2011/setting2'
Y_train_dev = pkl.load(gzip.open(os.path.join(base_dir, 'Y_train_dev.pkl.gz'), 'rb'))
PU_test = pkl.load(gzip.open(os.path.join(base_dir, 'PU_test.pkl.gz'), 'rb'))
In [ ]:
ftrain1 = os.path.join(base_dir, 'mf-train_p1.csv')
ftest = os.path.join(base_dir, 'mf-test.csv')
In [ ]:
Y_train_dev.shape
In [ ]:
PU_test.sum()
In [ ]:
lines = []
for i in range(PU_test.shape[0]):
if (i+1) % 1000 == 0:
sys.stdout.write('\r%d / %d' % (i+1, PU_test.shape[0])); sys.stdout.flush()
nzcol = PU_test[i, :].nonzero()[0]
lines += [','.join([ustrs[i], istrs2[j], '5\n']) for j in nzcol]
In [ ]:
len(lines)
In [ ]:
lines[0]
In [ ]:
with open(ftrain1, 'w') as fd:
fd.writelines(lines)
In [ ]:
ustrs = ['U%d' % i for i in range(Y_train_dev.shape[0])]
istrs1 = ['P%d' % j for j in range(Y_train_dev.shape[1])]
In [ ]:
lines = []
for i in range(Y_train_dev.shape[0]):
if (i+1) % 10 == 0:
sys.stdout.write('\r%d / %d' % (i+1, Y_train_dev.shape[0])); sys.stdout.flush()
# convert True to rating 5 and False to 1
lines += [','.join([ustrs[i], istrs1[j], '5\n']) if Y_train_dev[i, j] is True else \
','.join([ustrs[i], istrs1[j], '1\n']) for j in range(Y_train_dev.shape[1])]
In [ ]:
with open(ftrain, 'w') as fd:
for i in range(Y_train_dev.shape[0]):
sys.stdout.write('\r%d / %d' % (i+1, Y_train_dev.shape[0])); sys.stdout.flush()
istr = 'U%d' % i
for j in range(Y_train_dev.shape[1]):
#print(Y_train[i, j])
jstr = 'P%d' % j
v = Y_train_dev[i, j]
rating = 1 # convert False to rating 1
if v is True:
rating = 5 # convert True to rating 5
line = '%s,%s,%d\n' % (istr, jstr, rating)
#print(line)
fd.write(line)
In [ ]:
rows = ['LR', 'PC', 'LR-2017']
cols = ['F1', 'Precision@K']
df = pd.DataFrame(index=rows, columns=cols)
df.head()
In [ ]:
C = 1
#fname = os.path.join('data', 'aotm2011-params-br/br-aotm2011-C-%s.pkl' % str(C))
fname = os.path.join(data_dir, 'br-aotm2011-C-%g.pkl' % C)
br = pkl.load(open(fname, 'rb'))
Evaluate F1: threshold for logistic regression is 0 for logits, 0.5 for probabilities.
In [ ]:
F1 = evaluate_minibatch(br, calc_F1, X_dev, Y_dev, threshold=0, batch_size=1500, verbose=1)
avgF1 = np.mean(F1)
F1_all.append(avgF1)
print('\nF1: %g' % avgF1)
In [ ]:
np.mean(F1)
C: 0.1, Threshold: 0.05, F1: 0.00254648
C: 1, Threshold: 0.05, F1: 0.0121401
Evaluate Precision@K.
In [ ]:
pak = evaluate_minibatch(br, calc_precisionK, X_dev, Y_dev, threshold=None, batch_size=1500, verbose=1)
print('\nPrecision@K: %g' % np.mean(pak))
C: 0.1, Precision@K: 0.0884917
C: 1, Precision@K: 0.0943461
In [ ]:
best_C = 1
best_TH = 0.05
fname = os.path.join('data', 'aotm2011-params-br/br-aotm2011-C-%s.pkl' % str(best_C))
best_br = pkl.load(open(fname, 'rb'))
In [ ]:
F1_test_br = evaluate_minibatch(best_br, calc_F1, X_test, Y_test, threshold=best_TH, batch_size=1500, verbose=1)
print('\nTest F1: %g' % np.mean(F1_test_br))
In [ ]:
pak_test_br = evaluate_minibatch(best_br, calc_precisionK, X_test, Y_test,threshold=None,batch_size=1500,verbose=1)
print('\nTest Precision@K: %g' % np.mean(pak_test_br))
In [ ]:
df.loc['LR', 'F1'] = np.mean(F1_test_br)
df.loc['LR', 'Precision@K'] = np.mean(pak_test_br)
In [ ]:
df
In [ ]:
C_set = [0.1, 0.3, 1, 3, 10, 30, 100, 300, 1000, 3000, 10000, 30000]
p_set = [1, 2, 3, 4, 5, 6, 7, 8]
metrics_pc = [ ]
print('%15s %15s %15s %15s %15s' % ('C', 'p', 'Threshold', 'F1', 'Precision@K'))
for C in C_set:
for p in p_set:
#fname = os.path.join('data', 'aotm2011-params-pc/pc-aotm2011-C-%g-p-%g.pkl' % (C, p))
fname = os.path.join(data_dir, 'pc-aotm2011-C-%g-p-%g.pkl' % (C, p))
if not os.path.exists(fname): continue
pc_dict = pkl.load(open(fname, 'rb'))
print('%15s %15s %15s %15s %15s' % ('%g'%pc_dict['C'], '%g'%pc_dict['p'], \
'%g'%pc_dict['Threshold'], '%g'%pc_dict['F1'], \
'%g'%pc_dict['Precision@K']))
metrics_pc.append((pc_dict['C'], pc_dict['p'], pc_dict['Threshold'],pc_dict['F1'],pc_dict['Precision@K']))
clf = PClassificationMLC()
clf.load_params(fname)
th = pc_dict['Threshold']
F1 = evaluate_minibatch(clf, calc_F1, X_test, Y_test, threshold=th, batch_size=1500, verbose=1)
print('\nTest F1: %g' % np.mean(F1))
pak = evaluate_minibatch(clf, calc_precisionK, X_test, Y_test, threshold=None, batch_size=1500, verbose=1)
print('\nTest Precision@K: %g' % np.mean(pak))
In [ ]:
keyix = 3 # F1
sorted_metrics_pc = sorted(metrics_pc, key=lambda x: x[keyix], reverse=True)
print('Best hyper-param:\n(C, p, Threshold, F1, Precision@K):', sorted_metrics_pc[0])
In [ ]:
best_C = 30000 #10000 #300 #3000
best_p = 2 #2 #3 #6
best_TH = 0.1 #0.15 #0.1
#fname = os.path.join('data', 'aotm2011-params-pc/pc-aotm2011-C-%g-p-%g.pkl' % (best_C, best_p))
fname = os.path.join(data_dir, 'pc-aotm2011-C-%g-p-%g.pkl' % (best_C, best_p))
best_pc = PClassificationMLC()
best_pc.load_params(fname)
In [ ]:
bestdict = pkl.load(open(fname, 'rb'))
In [ ]:
plt.plot(bestdict['cost'])
In [ ]:
F1_test_pc = evaluate_minibatch(best_pc, calc_F1, X_test, Y_test, threshold=best_TH, batch_size=1500, verbose=1)
print('\nTest F1: %g' % np.mean(F1_test_pc))
In [ ]:
pak = evaluate_minibatch(best_pc, calc_precisionK, X_train, Y_train, threshold=None, batch_size=1500, verbose=1)
In [ ]:
print('\nTrain P@K: %g' % np.mean(pak))
In [ ]:
preds = best_pc.decision_function(X_train[:10])
In [ ]:
test_ex_idx = 2
plt.hist(preds[test_ex_idx], bins=50)
y_true = Y_train[test_ex_idx].toarray()
pos_idx = np.where(y_true)[1]
print('prediction of true positives')
print(preds[test_ex_idx][pos_idx])
print('top predictions')
np.sort(preds[test_ex_idx])[-20:]
In [ ]:
pak_test_pc = evaluate_minibatch(best_pc, calc_precisionK, X_test, Y_test,threshold=None,batch_size=1500,verbose=1)
print('\nTest Precision@K: %g' % np.mean(pak_test_pc))
In [ ]:
df.loc['PC', 'F1'] = np.mean(F1_test_pc)
df.loc['PC', 'Precision@K'] = np.mean(pak_test_pc)
In [ ]:
df.loc['LR-2017', 'F1'] = 0.031
In [ ]:
tab_str = df.to_latex(float_format=lambda x: '$%.4f$' % x, na_rep='-', multirow=False, escape=False)
In [ ]:
print('\\begin{table}[!h]')
print('\centering')
print('\\caption{Performance on test set}')
print('\\label{tab:perf}')
print(tab_str)
print('\\end{table}')