In [ ]:
%matplotlib inline
import os
import sys
import gzip
import numpy as np
import pickle as pkl
from scipy.sparse import lil_matrix, issparse, hstack, vstack
from collections import Counter
import gensim
import matplotlib.pyplot as plt
import seaborn as sns
In [ ]:
np_settings0 = np.seterr(all='raise')
RAND_SEED = 0
plt.style.use('seaborn')
In [ ]:
datasets = ['aotm2011', '30music']
ffeature = 'data/msd/song2feature.pkl.gz'
fgenre = 'data/msd/song2genre.pkl.gz'
fsong2artist = 'data/msd/song2artist.pkl.gz'
audio_feature_indices = [20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 185, 186, 187, 198, 199, 200, 201]
In [ ]:
dix = 0
dataset_name = datasets[dix]
data_dir = 'data/%s' % dataset_name
print(dataset_name)
Load playlists.
In [ ]:
fplaylist = os.path.join(data_dir, '%s-playlist.pkl.gz' % dataset_name)
_all_playlists = pkl.load(gzip.open(fplaylist, 'rb'))
In [ ]:
# _all_playlists[0]
In [ ]:
all_playlists = []
if type(_all_playlists[0][1]) == tuple:
for pl, u in _all_playlists:
user = '%s_%s' % (u[0], u[1]) # user string
all_playlists.append((pl, user))
else:
all_playlists = _all_playlists
In [ ]:
# user_playlists = dict()
# for pl, u in all_playlists:
# try:
# user_playlists[u].append(pl)
# except KeyError:
# user_playlists[u] = [pl]
In [ ]:
# all_playlists = []
# for u in user_playlists:
# if len(user_playlists[u]) > 4:
# all_playlists += [(pl, u) for pl in user_playlists[u]]
In [ ]:
all_users = sorted(set({user for _, user in all_playlists}))
In [ ]:
print('#user : {:,}'.format(len(all_users)))
print('#playlist: {:,}'.format(len(all_playlists)))
In [ ]:
pl_lengths = [len(pl) for pl, _ in all_playlists]
plt.hist(pl_lengths, bins=100)
print('Average playlist length: %.1f' % np.mean(pl_lengths))
check duplicated songs in the same playlist.
In [ ]:
print('{:,} | {:,}'.format(np.sum(pl_lengths), np.sum([len(set(pl)) for pl, _ in all_playlists])))
Load song_id --> feature array mapping: map a song to the audio features of one of its corresponding tracks in MSD.
In [ ]:
_song2feature = pkl.load(gzip.open(ffeature, 'rb'))
In [ ]:
song2feature = dict()
for sid in sorted(_song2feature):
song2feature[sid] = _song2feature[sid][audio_feature_indices]
Song genres from MSD Allmusic Genre Dataset (Top MAGD) and tagtraum.
In [ ]:
song2genre = pkl.load(gzip.open(fgenre, 'rb'))
In [ ]:
_all_songs = sorted([(sid, int(song2feature[sid][-1])) for sid in {s for pl, _ in all_playlists for s in pl}],
key=lambda x: (x[1], x[0]))
print('{:,}'.format(len(_all_songs)))
Randomise the order of song with the same age.
In [ ]:
song_age_dict = dict()
for sid, age in _all_songs:
age = int(age)
try:
song_age_dict[age].append(sid)
except KeyError:
song_age_dict[age] = [sid]
In [ ]:
all_songs = []
np.random.seed(RAND_SEED)
for age in sorted(song_age_dict.keys()):
all_songs += [(sid, age) for sid in np.random.permutation(song_age_dict[age])]
In [ ]:
pkl.dump(all_songs, gzip.open(os.path.join(data_dir, 'setting2/all_songs.pkl.gz'), 'wb'))
Check if all songs have genre info.
In [ ]:
print('#songs missing genre: {:,}'.format(len(all_songs) - np.sum([sid in song2genre for (sid, _) in all_songs])))
Song popularity.
In [ ]:
song2index = {sid: ix for ix, (sid, _) in enumerate(all_songs)}
song_pl_mat = lil_matrix((len(all_songs), len(all_playlists)), dtype=np.int8)
for j in range(len(all_playlists)):
pl = all_playlists[j][0]
ind = [song2index[sid] for sid in pl]
song_pl_mat[ind, j] = 1
In [ ]:
song_pop = song_pl_mat.tocsc().sum(axis=1)
In [ ]:
max_pop = np.max(song_pop)
max_pop
In [ ]:
song2pop = {sid: song_pop[song2index[sid], 0] for (sid, _) in all_songs}
In [ ]:
pkl.dump(song2pop, gzip.open(os.path.join(data_dir, 'setting2/song2pop.pkl.gz'), 'wb'))
deal with one outlier.
In [ ]:
# song_pop1 = song_pop.copy()
# maxix = np.argmax(song_pop)
# song_pop1[maxix] = 0
# clipped_max_pop = np.max(song_pop1) + 10 # second_max_pop + 10
# if max_pop - clipped_max_pop > 500:
# song_pop1[maxix] = clipped_max_pop
Songs as rows, playlists as columns.
In [ ]:
def gen_dataset(playlists, song2feature, song2genre, song2artist, artist2vec,
train_song_set, dev_song_set=[], test_song_set=[], song2pop_train=None):
"""
Create labelled dataset: rows are songs, columns are users.
Input:
- playlists: a set of playlists
- train_song_set: a list of songIDs in training set
- dev_song_set: a list of songIDs in dev set
- test_song_set: a list of songIDs in test set
- song2feature: dictionary that maps songIDs to features from MSD
- song2genre: dictionary that maps songIDs to genre
- song2pop_train: a dictionary that maps songIDs to its popularity
Output:
- (Feature, Label) pair (X, Y)
X: #songs by #features
Y: #songs by #users
"""
song_set = train_song_set + dev_song_set + test_song_set
N = len(song_set)
K = len(playlists)
genre_set = sorted({v for v in song2genre.values()})
genre2index = {genre: ix for ix, genre in enumerate(genre_set)}
def onehot_genre(songID):
"""
One-hot encoding of genres.
Data imputation:
- mean imputation (default)
- one extra entry for songs without genre info
- sampling from the distribution of genre popularity
"""
num = len(genre_set) # + 1
vec = np.zeros(num, dtype=np.float)
if songID in song2genre:
genre_ix = genre2index[song2genre[songID]]
vec[genre_ix] = 1
else:
vec[:] = np.nan
#vec[-1] = 1
return vec
def song_artist_feature(songID):
"""
Return the artist feature for a given song
"""
if songID in song2artist:
aid = song2artist[songID]
return artist2vec[aid]
else:
return artist2vec['$UNK$']
X = np.array([np.concatenate([song2feature[sid], song_artist_feature(sid), onehot_genre(sid)], axis=-1) \
for sid in song_set])
Y = lil_matrix((N, K), dtype=np.bool)
song2index = {sid: ix for ix, sid in enumerate(song_set)}
for k in range(K):
pl = playlists[k]
indices = [song2index[sid] for sid in pl if sid in song2index]
Y[indices, k] = True
# genre imputation
genre_ix_start = -len(genre_set)
genre_nan = np.isnan(X[:, genre_ix_start:])
genre_mean = np.nansum(X[:, genre_ix_start:], axis=0) / (X.shape[0] - np.sum(genre_nan, axis=0))
#print(np.nansum(X[:, genre_ix_start:], axis=0))
#print(genre_set)
#print(genre_mean)
for j in range(len(genre_set)):
X[genre_nan[:,j], j+genre_ix_start] = genre_mean[j]
# normalise the sum of all genres per song to 1
# X[:, -len(genre_set):] /= X[:, -len(genre_set):].sum(axis=1).reshape(-1, 1)
# NOTE: this is not necessary, as the imputed values are guaranteed to be normalised (sum to 1)
# due to the above method to compute mean genres.
# the log of song popularity
if song2pop_train is not None:
# for sid in song_set:
# assert sid in song2pop_train # trust the input
logsongpop = np.log2([song2pop_train[sid]+1 for sid in song_set]) # deal with 0 popularity
X = np.hstack([X, logsongpop.reshape(-1, 1)])
#return X, Y
Y = Y.tocsr()
train_ix = [song2index[sid] for sid in train_song_set]
X_train = X[train_ix, :]
Y_train = Y[train_ix, :]
dev_ix = [song2index[sid] for sid in dev_song_set]
X_dev = X[dev_ix, :]
Y_dev = Y[dev_ix, :]
test_ix = [song2index[sid] for sid in test_song_set]
X_test = X[test_ix, :]
Y_test = Y[test_ix, :]
if len(dev_song_set) > 0:
if len(test_song_set) > 0:
return X_train, Y_train.tocsc(), X_dev, Y_dev.tocsc(), X_test, Y_test.tocsc()
else:
return X_train, Y_train.tocsc(), X_dev, Y_dev.tocsc()
else:
if len(test_song_set) > 0:
return X_train, Y_train.tocsc(), X_test, Y_test.tocsc()
else:
return X_train, Y_train.tocsc()
Split playlists such that every song in test set is also in training set.
Split playlists (60/10/30 train/dev/test split) uniformly at random.
Split each user's playlists (60/20/20 train/dev/test split) uniformly at random if the user has $5$ or more playlists.
In [ ]:
train_playlists = []
dev_playlists = []
test_playlists = []
In [ ]:
candidate_pl_indices = []
other_pl_indices = []
for i in range(len(all_playlists)):
pl = all_playlists[i][0]
if np.all(np.asarray([song2pop[sid] for sid in pl]) >= 5):
candidate_pl_indices.append(i)
else:
other_pl_indices.append(i)
In [ ]:
print('%d + %d = %d | %d' % (len(candidate_pl_indices), len(other_pl_indices), \
len(candidate_pl_indices) + len(other_pl_indices), len(all_playlists)))
In [ ]:
dev_ratio = 0.05
test_ratio = 0.2
npl_dev = int(dev_ratio * len(all_playlists))
npl_test = int(test_ratio * len(all_playlists))
np.random.seed(RAND_SEED)
pl_indices = np.random.permutation(candidate_pl_indices)
test_ix = pl_indices[:npl_test]
test_playlists = [all_playlists[ix] for ix in test_ix]
dev_ix = pl_indices[npl_test:npl_test + npl_dev]
dev_playlists = [all_playlists[ix] for ix in dev_ix]
train_ix = pl_indices[npl_test + npl_dev:].tolist() + other_pl_indices
train_playlists = [all_playlists[ix] for ix in train_ix]
Every song in test set should also be in training set.
In [ ]:
print('#Songs in train + dev set: %d, #Songs total: %d' % \
(len(set([sid for pl, _ in train_playlists + dev_playlists for sid in pl])), len(all_songs)))
In [ ]:
print('{:30s} {:,}'.format('#playlist in training set:', len(train_playlists)))
print('{:30s} {:,}'.format('#playlist in dev set:', len(dev_playlists)))
print('{:30s} {:,}'.format('#playlist in test set:', len(test_playlists)))
In [ ]:
len(train_playlists) + len(dev_playlists)
In [ ]:
# user_playlists = dict()
# for pl, u in all_playlists:
# try:
# user_playlists[u].append(pl)
# except KeyError:
# user_playlists[u] = [pl]
In [ ]:
# sanity check
# npl_all = np.sum([len(user_playlists[u]) for u in user_playlists])
# print('{:30s} {:,}'.format('#users:', len(user_playlists)))
# print('{:30s} {:,}'.format('#playlists:', npl_all))
# print('{:30s} {:.2f}'.format('Average #playlists per user:', npl_all / len(user_playlists)))
In [ ]:
# np.random.seed(RAND_SEED)
# for u in user_playlists:
# playlists_u = [(pl, u) for pl in user_playlists[u]]
# if len(user_playlists[u]) < 5:
# train_playlists += playlists_u
# else:
# npl_test = int(test_ratio * len(user_playlists[u]))
# npl_dev = int(dev_ratio * len(user_playlists[u]))
# pl_indices = np.random.permutation(len(user_playlists[u]))
# test_playlists += playlists_u[:npl_test]
# dev_playlists += playlists_u[npl_test:npl_test + npl_dev]
# train_playlists += playlists_u[npl_test + npl_dev:]
In [ ]:
xmax = np.max([len(pl) for (pl, _) in all_playlists]) + 1
In [ ]:
ax = plt.subplot(111)
ax.hist([len(pl) for (pl, _) in train_playlists], bins=100)
ax.set_yscale('log')
ax.set_xlim(0, xmax)
ax.set_title('Histogram of playlist length in TRAINING set')
pass
In [ ]:
ax = plt.subplot(111)
ax.hist([len(pl) for (pl, _) in dev_playlists], bins=100)
ax.set_yscale('log')
ax.set_xlim(0, xmax)
ax.set_title('Histogram of playlist length in DEV set')
pass
In [ ]:
ax = plt.subplot(111)
ax.hist([len(pl) for (pl, _) in test_playlists], bins=100)
ax.set_yscale('log')
ax.set_xlim(0, xmax)
ax.set_title('Histogram of playlist length in TEST set')
pass
In [ ]:
song2artist = pkl.load(gzip.open(fsong2artist, 'rb'))
In [ ]:
artist_playlist = []
for pl, _ in train_playlists + dev_playlists:
pl_artists = [song2artist[sid] if sid in song2artist else '$UNK$' for sid in pl]
artist_playlist.append(pl_artists)
In [ ]:
fartist2vec_bin = os.path.join(data_dir, 'setting2/artist2vec.bin')
if os.path.exists(fartist2vec_bin):
artist2vec = gensim.models.KeyedVectors.load_word2vec_format(fartist2vec_bin, binary=True)
else:
artist2vec_model = gensim.models.Word2Vec(sentences=artist_playlist, size=50, seed=RAND_SEED,
window=10, iter=10, min_count=1)
artist2vec_model.wv.save_word2vec_format(fartist2vec_bin, binary=True)
artist2vec = artist2vec_model.wv
Keep the first $K=1,2,3,4$ songs for playlist in dev and test set.
In [ ]:
N_SEED_K = 1
In [ ]:
dev_playlists_obs = []
dev_playlists_held = []
test_playlists_obs = []
test_playlists_held = []
In [ ]:
for pl, _ in dev_playlists:
npl = len(pl)
k = N_SEED_K
dev_playlists_obs.append(pl[:k])
dev_playlists_held.append(pl[k:])
for pl, _ in test_playlists:
npl = len(pl)
k = N_SEED_K
test_playlists_obs.append(pl[:k])
test_playlists_held.append(pl[k:])
In [ ]:
for ix in range(len(dev_playlists)):
assert np.all(dev_playlists[ix][0] == dev_playlists_obs[ix] + dev_playlists_held[ix])
for ix in range(len(test_playlists)):
assert np.all(test_playlists[ix][0] == test_playlists_obs[ix] + test_playlists_held[ix])
In [ ]:
print('DEV obs: {:,} | DEV held: {:,} \nTEST obs: {:,} | TEST held: {:,}'.format(
np.sum([len(ppl) for ppl in dev_playlists_obs]), np.sum([len(ppl) for ppl in dev_playlists_held]),
np.sum([len(ppl) for ppl in test_playlists_obs]), np.sum([len(ppl) for ppl in test_playlists_held])))
In [ ]:
song2pop_train = song2pop.copy()
song2pop_trndev = song2pop.copy()
for ppl in dev_playlists_held:
for sid in ppl:
song2pop_train[sid] -= 1
for ppl in test_playlists_held:
for sid in ppl:
song2pop_train[sid] -= 1
song2pop_trndev[sid] -= 1
In [ ]:
pkl_dir2 = os.path.join(data_dir, 'setting2')
fpl2 = os.path.join(pkl_dir2, 'playlists_train_dev_test_s2_%d.pkl.gz' % N_SEED_K)
fy2 = os.path.join(pkl_dir2, 'Y_%d.pkl.gz' % N_SEED_K)
fxtrain2 = os.path.join(pkl_dir2, 'X_train_%d.pkl.gz' % N_SEED_K)
fytrain2 = os.path.join(pkl_dir2, 'Y_train_%d.pkl.gz' % N_SEED_K)
fxtrndev2 = os.path.join(pkl_dir2, 'X_trndev_%d.pkl.gz' % N_SEED_K)
fytrndev2 = os.path.join(pkl_dir2, 'Y_trndev_%d.pkl.gz' % N_SEED_K)
fydev2 = os.path.join(pkl_dir2, 'PU_dev_%d.pkl.gz' % N_SEED_K)
fytest2 = os.path.join(pkl_dir2, 'PU_test_%d.pkl.gz' % N_SEED_K)
fclique21 = os.path.join(pkl_dir2, 'cliques_trndev_%d.pkl.gz' % N_SEED_K)
fclique22 = os.path.join(pkl_dir2, 'cliques_all_%d.pkl.gz' % N_SEED_K)
In [ ]:
X, Y = gen_dataset(playlists = [t[0] for t in train_playlists + dev_playlists + test_playlists],
song2feature = song2feature, song2genre = song2genre,
song2artist = song2artist, artist2vec = artist2vec,
train_song_set = [t[0] for t in all_songs], song2pop_train=song2pop_train)
In [ ]:
X_train = X
assert X.shape[0] == len(song2pop_trndev)
X_trndev = X_train.copy()
X_trndev[:, -1] = np.log([song2pop_trndev[sid]+1 for sid, _ in all_songs])
In [ ]:
dev_cols = np.arange(len(train_playlists), len(train_playlists) + len(dev_playlists))
test_cols = np.arange(len(train_playlists) + len(dev_playlists), Y.shape[1])
assert len(dev_cols) == len(dev_playlists) == len(dev_playlists_obs)
assert len(test_cols) == len(test_playlists) == len(test_playlists_obs)
In [ ]:
pkl.dump({'train_playlists': train_playlists, 'dev_playlists': dev_playlists, 'test_playlists': test_playlists,
'dev_playlists_obs': dev_playlists_obs, 'dev_playlists_held': dev_playlists_held,
'test_playlists_obs': test_playlists_obs, 'test_playlists_held': test_playlists_held},
gzip.open(fpl2, 'wb'))
In [ ]:
song2index = {sid: ix for ix, sid in enumerate([t[0] for t in all_songs])}
Use dedicated sparse matrices to reprsent what entries are observed in dev and test set.
In [ ]:
Y_train = Y[:, :len(train_playlists)].tocsc()
Y_trndev = Y[:, :len(train_playlists) + len(dev_playlists)].tocsc()
In [ ]:
PU_dev = lil_matrix((len(all_songs), len(dev_playlists)), dtype=np.bool)
PU_test = lil_matrix((len(all_songs), len(test_playlists)), dtype=np.bool)
num_known_dev = 0
for j in range(len(dev_playlists)):
if (j+1) % 1000 == 0:
sys.stdout.write('\r%d / %d' % (j+1, len(dev_playlists))); sys.stdout.flush()
rows = [song2index[sid] for sid in dev_playlists_obs[j]]
PU_dev[rows, j] = True
num_known_dev += len(rows)
num_known_test = 0
for j in range(len(test_playlists)):
if (j+1) % 1000 == 0:
sys.stdout.write('\r%d / %d' % (j+1, len(test_playlists))); sys.stdout.flush()
rows = [song2index[sid] for sid in test_playlists_obs[j]]
PU_test[rows, j] = True
num_known_test += len(rows)
PU_dev = PU_dev.tocsr()
PU_test = PU_test.tocsr()
In [ ]:
print('#unknown entries in DEV set: {:15,d} | {:15,d} \n#unknown entries in TEST set: {:15,d} | {:15,d}'.format(
np.prod(PU_dev.shape) - PU_dev.sum(), len(dev_playlists) * len(all_songs) - num_known_dev,
np.prod(PU_test.shape) - PU_test.sum(), len(test_playlists) * len(all_songs) - num_known_test))
Feature normalisation.
In [ ]:
X_train_mean = np.mean(X_train, axis=0).reshape((1, -1))
X_train_std = np.std(X_train, axis=0).reshape((1, -1)) + 10 ** (-6)
X_train -= X_train_mean
X_train /= X_train_std
In [ ]:
X_trndev_mean = np.mean(X_trndev, axis=0).reshape((1, -1))
X_trndev_std = np.std(X_trndev, axis=0).reshape((1, -1)) + 10 ** (-6)
X_trndev -= X_trndev_mean
X_trndev /= X_trndev_std
In [ ]:
print(np.mean(np.mean(X_train, axis=0)))
print(np.mean( np.std(X_train, axis=0)) - 1)
print(np.mean(np.mean(X_trndev, axis=0)))
print(np.mean( np.std(X_trndev, axis=0)) - 1)
In [ ]:
print('All : %s' % str(Y.shape))
print('Train : %s, %s' % (X_train.shape, Y_train.shape))
print('Dev : %s' % str(PU_dev.shape))
print('Trndev: %s, %s' % (X_trndev.shape, Y_trndev.shape))
print('Test : %s' % str(PU_test.shape))
In [ ]:
pkl.dump(X_train, gzip.open(fxtrain2, 'wb'))
pkl.dump(Y_train, gzip.open(fytrain2, 'wb'))
pkl.dump(Y, gzip.open(fy2, 'wb'))
pkl.dump(X_trndev, gzip.open(fxtrndev2, 'wb'))
pkl.dump(Y_trndev, gzip.open(fytrndev2, 'wb'))
pkl.dump(PU_dev, gzip.open(fydev2, 'wb'))
pkl.dump(PU_test, gzip.open(fytest2, 'wb'))
Build the adjacent matrix of playlists (nodes) for setting II, playlists of the same user form a clique.
Cliques in train + dev set.
In [ ]:
pl_users = [u for (_, u) in train_playlists + dev_playlists]
cliques_trndev = []
for u in sorted(set(pl_users)):
clique = np.where(u == np.array(pl_users, dtype=np.object))[0]
#if len(clique) > 1:
cliques_trndev.append(clique)
In [ ]:
pkl.dump(cliques_trndev, gzip.open(fclique21, 'wb'))
In [ ]:
clqsize = [len(clq) for clq in cliques_trndev]
print(np.min(clqsize), np.max(clqsize), len(clqsize), np.sum(clqsize))
In [ ]:
assert np.all(np.arange(Y_trndev.shape[1]) == np.asarray(sorted([k for clq in cliques_trndev for k in clq])))
Cliques in train + dev + test set.
In [ ]:
pl_users = [u for (_, u) in train_playlists + dev_playlists + test_playlists]
clique_all = []
for u in sorted(set(pl_users)):
clique = np.where(u == np.array(pl_users, dtype=np.object))[0]
#if len(clique) > 1:
clique_all.append(clique)
In [ ]:
pkl.dump(clique_all, gzip.open(fclique22, 'wb'))
In [ ]:
clqsize = [len(clq) for clq in clique_all]
print(np.min(clqsize), np.max(clqsize), len(clqsize), np.sum(clqsize))
In [ ]:
assert np.all(np.arange(Y.shape[1]) == np.asarray(sorted([k for clq in clique_all for k in clq])))