In [97]:
import cubehelix
from joblib import Parallel, delayed
import matplotlib
import matplotlib.pyplot as plt
from matplotlib import gridspec
import numpy as np
import os
import pandas as pd
import seaborn as sns
import msaf # Develop branch
import mir_eval # Uri's fork: https://github.com/urinieto/mir_eval
from mir_eval.segment import tree
# Beatuiful seaborn plots
sns.set(style="white")
%matplotlib inline
# Colorhelix config
cx = cubehelix.cmap(reverse=True, start=0.2, rot=-0.4, minLight=0.45, maxLight=1.0)
# SALAMI Path
SALAMI_path = "/home/uri/datasets/Segments/references/"
SALAMI_path = "/Users/uriadmin/datasets/Segments/references/"
In [103]:
def gen_synthetic_labels(bounds):
labels = []
[labels.append(np.random.random(len(level) - 1)) for level in bounds]
return labels
def evaluate_flat(ref_times, est_times, w, trim=False):
ref_times = np.asarray(ref_times).flatten()
est_times = np.asarray(est_times).flatten()
ref_inter = np.asarray(zip(ref_times[:], ref_times[1:]))
est_inter = np.asarray(zip(est_times[:], est_times[1:]))
p, r, f = mir_eval.segment.detection(ref_inter, est_inter, trim=trim, window=w)
R2E, E2R = mir_eval.segment.deviation(ref_inter, est_inter, trim=trim)
print "F: %.2f\t P: %.2f\t R: %.2f" % (f * 100, p * 100, r * 100)
print "R2E: %.2f\t E2R: %.2f" % (R2E, E2R)
return f, p, r
def evaluate_hier(bounds1, bounds2, levels1=None, levels2=None, title=None, figsize=(7,3),
ratios=[1,1], plot=True):
labels1 = gen_synthetic_labels(bounds1)
labels2 = gen_synthetic_labels(bounds2)
tree1 = tree.SegmentTree(bounds1, labels1, levels1)
tree2 = tree.SegmentTree(bounds2, labels2, levels2)
res = 0.1
for ws in [0.5, 3.0, 15, 30, None]:
if ws is None:
w = None
else:
w = int(ws / res)
T_o, T_u, T_f = mir_eval.segment.hmeasure(tree1, tree2, transitive=False, window=w, res=res)
if ws is None:
ws = np.inf
print "w: %.1f\t Tu: %.2f\t To: %.2f" % (ws, T_u * 100, T_o * 100)
if plot:
plt.figure(figsize=figsize)
gs = gridspec.GridSpec(2, 1, height_ratios=ratios)
plt.subplot(gs[0])
msaf.plotting.plot_tree(tree1, title="Reference", res=None, cmap_id=cx)
plt.subplot(gs[1])
msaf.plotting.plot_tree(tree2, title="Estimation", res=None, cmap_id=cx)
plt.tight_layout()
def remove_level(bounds, levels, level_id):
for i, level in enumerate(levels):
if level_id == level:
bounds.pop(i)
levels.pop(i)
break
In [3]:
# Synthetic example
est_bounds = [[0, 30, 100], [0, 10, 30, 40, 80, 100], [0, 10, 25, 30, 40, 50, 80, 95, 100]]
est_labels = gen_synthetic_labels(est_bounds)
est_tree = tree.SegmentTree(est_bounds, est_labels)
plt.figure(figsize=(7,2))
msaf.plotting.plot_tree(est_tree, title="Hierarchical Segmentation", res=None, cmap_id=cx)
plt.tight_layout()
plt.savefig("../figs/hier-example.pdf")
In [52]:
# Synthetic example (1)
orig_size = matplotlib.rcParams["font.size"]
matplotlib.rcParams["font.size"] = 14.
est_bounds1 = [[0, 10, 30], [0, 10, 20, 30]]
est_labels1 = gen_synthetic_labels(est_bounds1)
est_tree1 = tree.SegmentTree(est_bounds1, est_labels1)
est_bounds2 = [[0, 20, 30], [0, 10, 20, 30]]
est_labels2 = est_labels1
est_tree2 = tree.SegmentTree(est_bounds2, est_labels2)
plt.figure(figsize=(7,2))
plt.subplot(121)
msaf.plotting.plot_tree(est_tree1, title="", res=None, cmap_id=cx)
plt.xticks([],[])
plt.tight_layout()
plt.xlabel("Time")
# Annotate level 0
plt.gca().text(5, .75, 'A', horizontalalignment='center', verticalalignment='center')
plt.gca().text(20, .75, 'B', horizontalalignment='center', verticalalignment='center')
# Annotate level 1
plt.gca().text(5, .25, 'a', horizontalalignment='center', verticalalignment='center')
plt.gca().text(15, .25, 'b', horizontalalignment='center', verticalalignment='center')
plt.gca().text(25, .25, 'c', horizontalalignment='center', verticalalignment='center')
plt.subplot(122)
msaf.plotting.plot_tree(est_tree2, title="", res=None, cmap_id=cx)
plt.xticks([],[])
plt.yticks([],[])
plt.tight_layout()
plt.xlabel("Time")
# Annotate level 0
plt.gca().text(10, 0.75, 'A', horizontalalignment='center', verticalalignment='center')
plt.gca().text(25, 0.75, 'B', horizontalalignment='center', verticalalignment='center')
# Annotate level 1
plt.gca().text(5, .25, 'a', horizontalalignment='center', verticalalignment='center')
plt.gca().text(15, .25, 'b', horizontalalignment='center', verticalalignment='center')
plt.gca().text(25, .25, 'c', horizontalalignment='center', verticalalignment='center')
plt.savefig("../figs/toy-examples.pdf")
matplotlib.rcParams["font.size"] = orig_size
In [4]:
# Synthetic example (2)
orig_size = matplotlib.rcParams["font.size"]
matplotlib.rcParams["font.size"] = 14.
est_bounds = [[0, 30, 40], [0, 10, 30, 40], [0, 10, 25, 30, 40]]
est_labels = gen_synthetic_labels(est_bounds)
est_tree = tree.SegmentTree(est_bounds, est_labels)
plt.figure(figsize=(7,2))
msaf.plotting.plot_tree(est_tree, title="Hierarchical Segmentation", res=None, cmap_id=cx)
plt.xticks([5, 17.5, 27.5, 35], ["$i$", "$j$", "$k$", "$m$"])
plt.tight_layout()
plt.xlabel("Time")
# Annotate level 2
plt.gca().text(5, 0.165, '$H(i,i)$', horizontalalignment='center', verticalalignment='center')
plt.gca().text(17.5, 0.165, '$H(j,j)$', horizontalalignment='center', verticalalignment='center')
plt.gca().text(27.5, 0.165, '$H(k,k)$', horizontalalignment='center', verticalalignment='center')
plt.gca().text(35, 0.165, '$H(m,m)$', horizontalalignment='center', verticalalignment='center')
# Annotate level 1
plt.gca().text(20, 0.5, '$H(j,k)$', horizontalalignment='center', verticalalignment='center')
# Annotate level 0
plt.gca().text(15, .845, '$H(i,k)$', horizontalalignment='center', verticalalignment='center')
plt.savefig("../figs/hier-example.pdf")
matplotlib.rcParams["font.size"] = orig_size
In [5]:
# Flat vs flat
f1 = [[0, 10, 20, 30, 40, 50, 60]]
f2 = [[0, 20, 40, 60]]
f, p, r = evaluate_flat(f1, f2, 3, trim=True)
evaluate_hier(f1, f2, title="Flat vs Flat", figsize=(7,2.5))
plt.savefig("../figs/flat-flat.pdf")
In [6]:
#Hierarchical vs Flat Larger Scale
h1 = [[0, 20, 40, 60], [0, 10, 20, 30, 40, 50, 60]]
f2 = [[0, 40, 60]]
evaluate_hier(h1, f2, title="Hierarchical vs Flat Large Scale", figsize=(7,2.5))
plt.savefig("../figs/hier-flatlarger.pdf")
In [55]:
#Hierarchical vs Flat Large Scale
h1 = [[0, 20, 40, 60], [0, 10, 20, 30, 40, 50, 60]]
f2 = [[0, 20, 40, 60]]
evaluate_hier(h1, f2, title="Hierarchical vs Flat Large Scale", figsize=(7,2.5))
plt.savefig("../figs/hier-flatlarge.pdf")
In [56]:
# Hierarchical vs Flat Small Scale
h1 = [[0, 20, 40, 60], [0, 10, 20, 30, 40, 50, 60]]
f2 = [[0, 10, 20, 30, 40, 50, 60]]
evaluate_hier(h1, f2, title="Hierarchical vs Flat Small Scale", figsize=(7,2.5))
plt.savefig("../figs/hier-flatsmall.pdf")
In [43]:
# Hierarchical vs Flat Smaller Scale
h1 = [[0, 20, 40, 60], [0, 10, 20, 30, 40, 50, 60]]
f2 = [[0, 10, 15, 20, 25, 30, 35, 40, 45, 50, 60]]
evaluate_hier(h1, f2, title="Hierarchical vs Flat Small Scale", figsize=(7,2.5))
plt.savefig("../figs/hier-flatsmaller.pdf")
In [44]:
# Hierarchical vs Hierarchical
h1 = [[0, 20, 40, 60], [0, 10, 20, 30, 40, 50, 60]]
h2 = [[0, 40, 60], [0, 20, 40, 60], [0, 10, 20, 30, 40, 50, 60]]
evaluate_hier(h1, h2, title="Hierarchical vs Hierarchical", figsize=(7,3))
plt.savefig("../figs/hier-hiercomp.pdf")
In [57]:
jams_file = "SALAMI_636.jams"
hier_bounds1, hier_labels1, hier_levels1 = msaf.io.read_hier_references(SALAMI_path + jams_file, annotation_id=0)
hier_bounds2, hier_labels2, hier_levels2 = msaf.io.read_hier_references(SALAMI_path + jams_file, annotation_id=1)
# Remove the function level
hier_levels1 = list(hier_levels1)
hier_levels2 = list(hier_levels2)
remove_level(hier_bounds1, hier_levels1, "function")
remove_level(hier_bounds2, hier_levels2, "function")
# Evaluate
evaluate_hier(hier_bounds1, hier_bounds2, hier_levels1, hier_levels2, title="SALAMI vs SALAMI", figsize=(7,3))
plt.savefig("../figs/SALAMI-SALAMI.pdf")
In [106]:
jams_file = "SALAMI_636.jams"
olda_file = "/home/uri/Projects/hier_eval/test/SALAMI_636_OLDA.jams"
olda_file = "/Users/uriadmin/NYU/Spring14/hier_eval/test/SALAMI_636_OLDA.jams"
hier_bounds1, hier_labels1, hier_levels1 = msaf.io.read_hier_references(SALAMI_path + jams_file)
hier_bounds2, hier_labels2, hier_levels2 = msaf.io.read_hier_references(olda_file)
# Remove the function level from SALAMI
hier_levels1 = list(hier_levels1)
remove_level(hier_bounds1, hier_levels1, "function")
# Evaluate
evaluate_hier(hier_bounds1, hier_bounds2, hier_levels1, hier_levels2, title="SALAMI vs OLDA", figsize=(7,5), ratios=[1,3])
plt.savefig("../figs/SALAMI-OLDA.pdf")
In [20]:
# Automatically set w
def analyze_salami(salami_file, windows):
# Read annotations
try:
bounds1, labels1, levels1 = msaf.io.read_hier_references(salami_file.ref_file, annotation_id=0)
bounds2, labels2, levels2 = msaf.io.read_hier_references(salami_file.ref_file, annotation_id=1)
# Convert to tree
labels1 = gen_synthetic_labels(bounds1)
labels2 = gen_synthetic_labels(bounds2)
tree1 = tree.SegmentTree(bounds1, labels1, levels1)
tree2 = tree.SegmentTree(bounds2, labels2, levels2)
# Evaluate
print "Evaluating %s..." % salami_file.ref_file
res = 0.1
results = {}
results["file"] = os.path.basename(salami_file.ref_file)
for ws in windows:
if ws is None:
w = None
else:
w = int(ws / res)
T_o, T_u, T_f = mir_eval.segment.hmeasure(tree1, tree2, transitive=False, window=w, res=res)
if ws is None:
ws = np.inf
results["Tf_" + str(ws)] = T_f
results["To_" + str(ws)] = T_o
results["Tu_" + str(ws)] = T_u
except:
return []
return results
n_jobs = 8
salami_files = msaf.io.get_dataset_files("/home/uri/datasets/Segments/", ds_name="SALAMI")
windows = [0.5, 3, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60]
# Run in parallel
raw_results = Parallel(n_jobs=n_jobs)(delayed(analyze_salami)(salami_file, windows)
for salami_file in salami_files[:])
# Aggregate results in pandas format
results = pd.DataFrame()
for r in raw_results:
if r != []:
results = results.append(r, ignore_index=True)
In [21]:
print len(results)
print len(salami_files)
print results.mean()
print results.mean().values
print results.mean().keys()
In [22]:
results.to_csv("searching_w_nontransitive.csv")
print results.keys()
In [23]:
results = pd.read_csv("searching_w_nontransitive.csv")
windows = [0.5, 3, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60]
results.rename(columns={
'Tf_03': 'Tf_3', 'Tf_05': 'Tf_5',
'Tu_03': 'Tu_3', 'Tu_05': 'Tu_5',
'To_03': 'To_3', 'To_05': 'To_5'}, inplace=True)
In [26]:
Tf_keys = ["Tf_" + str(w) for w in windows]
To_keys = ["To_" + str(w) for w in windows]
Tu_keys = ["Tu_" + str(w) for w in windows]
plt.figure(figsize=(6,3))
plt.plot(windows, results[Tf_keys].mean().values, label="$T_F$")
plt.plot(windows, results[To_keys].mean().values, label="$T_o$")
plt.plot(windows, results[Tu_keys].mean().values, label="$T_u$")
plt.plot(windows[np.argmax(results[Tf_keys].mean().values)],
np.max(results[Tf_keys].mean().values), 'rD')
plt.ylim((0.4,1))
plt.legend(loc='lower right')
plt.xlabel("Window size (seconds)")
plt.ylabel("$T$-measures")
sns.despine()
plt.tight_layout()
plt.savefig("../figs/exploring_w.pdf")
plt.figure(figsize=(6,3))
plt.plot(windows, results[Tf_keys].mean().values, label="$T_F$")
plt.plot(windows[np.argmax(results[Tf_keys].mean().values)],
np.max(results[Tf_keys].mean().values), 'rD')
plt.ylim((0.4,1))
plt.xlabel("Window size (seconds)")
plt.ylabel("$T_F$ Score")
sns.despine()
plt.tight_layout()
plt.savefig("../figs/exploring_w_Tf.pdf")
In [ ]:
print results[Tf_keys].mean()
In [90]:
reload(msaf.utils)
ds_path = "/Users/uriadmin/datasets/Segments/"
file_name = "SALAMI_1200.jams"
file_name = "SALAMI_636.jams"
bound_id = "olda"
label_id = None
config = {'framesync': False,
'hier': True,
'annot_beats': False,
'transform': '/Users/uriadmin/NYU/Dissertation/msaf_github/msaf/algorithms/olda/models/EstBeats_BeatlesIso.npy',
'feature': 'hpcp'}
ref_times, ref_labels, ref_levels = \
msaf.io.read_hier_references(os.path.join(ds_path, msaf.Dataset.references_dir, file_name),
annotation_id=0,
exclude_levels=["function"])
est_inter, est_labels = msaf.io.read_estimations(os.path.join(ds_path, msaf.Dataset.estimations_dir, file_name),
bound_id, label_id, **config)
est_inter = sorted(est_inter, key=lambda level: len(level))
est_times = []
for inter in est_inter:
est_times.append(msaf.utils.intervals_to_times(inter))
est_labels = gen_synthetic_labels(est_times)
# Align the times
msaf.utils.align_end_hierarchies(est_times, ref_times)
# Build trees
ref_tree = mir_eval.segment.tree.SegmentTree(ref_times, ref_labels,
ref_levels)
est_tree = mir_eval.segment.tree.SegmentTree(est_times, est_labels)
msaf.utils.align_end_hierarchies(est_times, ref_times, thres=0.5)
In [107]:
# print ref_tree
reload(msaf.plotting)
reload(msaf.utils)
plt.figure(figsize=(7,2))
msaf.plotting.plot_tree(ref_tree, title="Hierarchical Segmentation", res=None, cmap_id=cx)
plt.figure(figsize=(7,2))
msaf.plotting.plot_tree(est_tree, title="Hierarchical Segmentation", res=None, cmap_id=cx)
print mir_eval.segment.hmeasure(ref_tree, est_tree, window=150)
In [104]:
evaluate_hier(ref_times, est_times, levels1=None, levels2=None, title=None, figsize=(7,3), ratios=[1,1], plot=True)