In [ ]:
import glob
import matplotlib
import msaf
import os
In [ ]:
def plot_scores(bound_scores, struct_scores, labels, title, out_pdf="out.pdf"):
metric_dict = {
"HitRate_t0.5F": "$\mathbf{F}_{0.5t}$",
"HitRate_t0.5P": "$\mathbf{P}_{0.5t}$",
"HitRate_t0.5R": "$\mathbf{R}_{0.5t}$",
"HitRate_0.5F": "$\mathbf{F}_{0.5}$",
"HitRate_0.5P": "$\mathbf{P}_{0.5}$",
"HitRate_0.5R": "$\mathbf{R}_{0.5}$",
"HitRate_t3F": "$\mathbf{F}_{3t}$",
"HitRate_t3P": "$\mathbf{P}_{3t}$",
"HitRate_t3R": "$\mathbf{R}_{3t}$",
"HitRate_3F": "$\mathbf{F}_{3}$",
"HitRate_3P": "$\mathbf{P}_{3}$",
"HitRate_3R": "$\mathbf{R}_{3}$",
"DevE2R": "$\mathbf{D}_{E2A}",
"DevR2E": "$\mathbf{D}_{A2E}",
"DevtE2R": "$\mathbf{D}_{E2At}",
"DevtR2E": "$\mathbf{D}_{A2Et}",
"PWF": "$\mathbf{Pw}_f$",
"PWP": "$\mathbf{Pw}_p$",
"PWR": "$\mathbf{Pw}_r$",
"Sf": "$\mathbf{S}_f$",
"So": "$\mathbf{S}_o$",
"Su": "$\mathbf{S}_u$"
}
def get_scores(res, order):
scores = [res[key] for key in order]
labels = [metric_dict[key] for key in order]
return scores, labels
def get_hitrate(res):
order = ["HitRate_t0.5F", "HitRate_t0.5P", "HitRate_t0.5R", "HitRate_0.5F", "HitRate_0.5P", "HitRate_0.5R",
"HitRate_t3F", "HitRate_t3P", "HitRate_t3R", "HitRate_3F", "HitRate_3P", "HitRate_3R"]
return get_scores(res, order)
def get_structural(res):
order = ["PWF", "PWP", "PWR", "Sf", "So", "Su"]
return get_scores(res, order)
# Plot Setup
matplotlib.rcParams.update({'font.size': 16})
plt.figure(figsize=(11,5))
plt.margins(0.1)
plt.ylabel("Scores")
plt.xlabel("Metrics")
plt.ylim((0, 1))
plt.title(title)
line_color = "k"
markers = ["ro--", "bs--", "gv--", "m*--", "cp--"]
# Get Plot Coordinates
y_axis = []
if bound_scores is None:
for i, struct_score in enumerate(struct_scores):
y, y_labels = get_structural(struct_score)
y_axis.append(y)
plt.axvline(2.5, color=line_color, alpha=0.3)
else:
for i, bound_score in enumerate(bound_scores):
y, y_labels = get_hitrate(bound_score)
if struct_scores is not None:
s_y, s_y_labels = get_structural(struct_scores[i])
y += s_y
y_labels += s_y_labels
y_axis.append(y)
plt.axvline(2.5, color=line_color, alpha=0.3)
plt.axvline(5.5, color=line_color, alpha=0.3)
plt.axvline(8.5, color=line_color, alpha=0.3)
if struct_scores is not None:
plt.axvline(11.5, color=line_color, alpha=0.3)
plt.axvline(14.5, color=line_color, alpha=0.3)
x = np.arange(len(y))
# Plot
plt.xticks(x, y_labels)
for y, label, marker in zip(y_axis, labels, markers):
plt.plot(x, y, marker, label=label)
plt.legend(loc=0)
plt.gca().yaxis.grid(True)
# Save
plt.savefig(out_pdf)
In [ ]:
def plot_devs(scores, labels, title, out_pdf="out.pdf"):
metric_dict = {
"DevE2R": "$\mathbf{D}_{E2A}$",
"DevR2E": "$\mathbf{D}_{A2E}$",
"DevtE2R": "$\mathbf{D}_{E2At}$",
"DevtR2E": "$\mathbf{D}_{A2Et}$"
}
def get_devs(res):
order = ["DevtE2R", "DevtR2E", "DevE2R", "DevR2E"]
scores = [res[key] for key in order]
labels = [metric_dict[key] for key in order]
return scores, labels
# Plot Setup
matplotlib.rcParams.update({'font.size': 16})
plt.figure(figsize=(11,5))
plt.margins(0.1)
plt.ylabel("Time (sec)")
plt.xlabel("Metrics")
plt.title(title)
line_color = "k"
plt.axvline(1.5, color=line_color, alpha=0.3)
markers = ["ro--", "bs--", "gv--", "m*--", "cp--"]
# Get Plot Coordinates
y_axis = []
for score in scores:
y, y_labels = get_devs(score)
y_axis.append(y)
x = np.arange(len(y))
# Plot
plt.xticks(x, y_labels)
for y, label, marker in zip(y_axis, labels, markers):
plt.plot(x, y, marker, label=label)
plt.ylim((0, int(max([max(y) for y in y_axis])) + 1))
plt.legend(loc=0)
plt.gca().yaxis.grid(True)
# Save
plt.savefig(out_pdf)
In [ ]:
ds_path = "/Users/uri/datasets/Segments/"
figures_path = "/Users/uri/Dropbox/NYU/Dissertation/manuscript/backmatter/figures/"
features_dict = {
"hpcp": "PCP",
"mfcc": "MFCC",
"tonnetz": "Tonnetz"
}
ds_dict = {
"Beatles": "ISO-Beatles",
"Cerulean": "Cerulean",
"Epiphyte": "Epyphyte",
"Isophonics": "Isophonics",
"SALAMI": "SALAMI",
"*": "All Datasets"
}
method_dict = {
"cc": "Constrained Cluster",
"cnmf3": "C-NMF",
"foote": "Checkerboard",
"sf": "Structural Features",
"siplca": "SI-PLCA",
"olda": "OLDA",
"fmc2d": "2D-FMC"
}
def plot_all(bid, ds_name, bound_scores, struct_scores, features_str):
title = "%s in %s" % (method_dict[bid], ds_dict[ds_name])
out_pdf = "%s-%s.pdf" % (bid, ds_name)
plot_scores(bound_scores, struct_scores, features_str, title, out_pdf=figures_path + "struct-" + out_pdf)
if bound_scores is not None:
plot_devs(bound_scores, features_str, title, out_pdf=figures_path + "devs-" + out_pdf)
In [ ]:
# Hit Rates + Structural
bids = ["cnmf3", "cc", "foote", "sf"]
features = ["hpcp", "mfcc", "tonnetz"]
ds_names = ds_dict.keys()
for bid in bids:
for ds_name in ds_names:
bound_scores = []
struct_scores = []
for feature in features:
bound_scores.append(msaf.eval.process(ds_path, bid, ds_name=ds_name, feature=feature, n_jobs=1, save=True).mean())
if bid == "cc" or bid == "cnmf3":
struct_scores.append(msaf.eval.process(ds_path, "gt", labels_id=bid, ds_name=ds_name, feature=feature, n_jobs=1, save=True).mean())
else:
struct_scores = None
features_str = [features_dict[feature] for feature in features]
plot_all(bid, ds_name, bound_scores, struct_scores, features_str)
In [ ]:
# 2D-FMC (only hpcp with K=5, 6, 7, and dirichlet. Only structural)
bid = "fmc2d"
feature = "hpcp"
ds_names = ds_dict.keys()
configs = [{"dirichlet": False, "k": 5}, {"dirichlet": False, "k": 6},
{"dirichlet": False, "k": 7}, {"dirichlet": True, "k": 6}]
for ds_name in ds_names:
struct_scores = []
for config in configs:
config["xmeans"] = False
struct_scores.append(msaf.eval.process(ds_path, "gt", labels_id=bid, ds_name=ds_name,
feature=feature, n_jobs=1, config=config, save=True).mean())
features_str = ["$K=5$", "$K=6$", "$K=7$", "Learned $K$"]
plot_all(bid, ds_name, None, struct_scores, features_str)
In [6]:
# OLDA (not All dataset, change models depending on dataset. Only boundaries)
bid = "olda"
feature = "hpcp"
ds_names = ["Beatles", "Isophonics", "Epiphyte", "SALAMI", "Cerulean"]
reload(msaf.eval)
for ds_name in ds_names:
if ds_name == "Epiphyte":
config = {"transform": "/Users/uriadmin/NYU/Dissertation/msaf/algorithms/olda/models/EstBeats_SALAMI.npy"}
features_str = ["Learned Features using SALAMI"]
else:
config = {"transform": "/Users/uriadmin/NYU/Dissertation/msaf/algorithms/olda/models/EstBeats_Epiphyte.npy"}
features_str = ["Learned Features using Epiphyte"]
bound_scores = [msaf.eval.process(ds_path, bid, ds_name=ds_name, feature=feature,
n_jobs=1, config=config, save=True).mean()]
plot_all(bid, ds_name, bound_scores, None, features_str)
In [ ]:
# SI-PLCA (only hpcp)
bid = "siplca"
feature = "hpcp"
ds_names = ds_dict.keys()
for ds_name in ds_names:
bound_scores = [msaf.eval.process(ds_path, bid, ds_name=ds_name, feature=feature,
n_jobs=1, save=True).mean()]
struct_scores = [msaf.eval.process(ds_path, "gt", labels_id=bid, ds_name=ds_name,
feature=feature, n_jobs=1, save=True).mean()]
features_str = [features_dict[feature]]
plot_all(bid, ds_name, bound_scores, struct_scores, features_str)