The purpose of this notebook is to plot the F-measure (or other metric) scores achieved for mock community, cross-validated, and novel taxa evaluations with each method/parameter configuration. Each configuration that is tested in all three evaluations will be plotted in 3D space as a single point (averaged across all datasets/samples, e.g., all samples in each mock community will be averaged since the same samples do not exist in the cv and nt evals).
First we load modules and data.
In [48]:
from os.path import join, expandvars
import pandas as pd
import matplotlib.pyplot as plt
import seaborn.xkcd_rgb as colors
from mpl_toolkits.mplot3d import Axes3D
from tax_credit.plotting_functions import isolate_top_params
#%matplotlib notebook
%matplotlib inline
In [2]:
project_dir = expandvars("../../")
precomputed_results = join(project_dir, "data", "precomputed-results")
mock_results = pd.DataFrame.from_csv(join(
precomputed_results, "mock-community", "mock_results.tsv"), sep="\t")
cv_results = pd.DataFrame.from_csv(join(
precomputed_results, "cross-validated", "evaluate_classification_summary.csv"))
nt_results = pd.DataFrame.from_csv(join(
precomputed_results, "novel-taxa-simulations", "evaluate_classification_summary.csv"))
# we can save plots in this directory
outdir = expandvars("../../../plots/")
We can set some global variables here
In [3]:
level = 6
metric = "F-measure"
method = "Method"
params = "Parameters"
reference = "Reference"
dataset = "Dataset"
color_palette={
'rdp': colors['baby shit green'], 'sortmerna': colors['macaroni and cheese'],
'uclust': 'coral', 'blast': 'indigo', 'blast+': colors['electric purple'], 'naive-bayes': 'dodgerblue',
'naive-bayes-bespoke': 'blue', 'vsearch': 'firebrick'
}
We will use the following functions for combining evaluations and plotting
In [181]:
def combined_evaluation(mock_results, cv_results, nt_results, params, metric, method, reference, dataset, domain, level):
pd.options.mode.chained_assignment = None # default='warn'
# Filter mock results to use correct datasets
dataset_ids = ['mock-1', 'mock-2', 'mock-3', 'mock-4', 'mock-5', 'mock-7', 'mock-8', 'mock-9',
'mock-10', 'mock-12', 'mock-16', 'mock-18', 'mock-19', 'mock-20', 'mock-21',
'mock-22', 'mock-23', 'mock-24', 'mock-26-ITS1', 'mock-26-ITS9']
method_ids = ['rdp', 'sortmerna', 'uclust', 'blast', 'blast+', 'naive-bayes', 'naive-bayes-bespoke', 'vsearch']
if domain == "fungi":
ref_ids = ['F1-REF', 'F2-REF', 'F3-REF', 'unite_20.11.2016_clean_fullITS']
elif domain == "bacteria":
ref_ids = ['B1-REF', 'B2-REF', 'B3-REF', 'gg_13_8_otus']
# otherwise combine bacteria and fungi
else:
ref_ids = ['B1-REF', 'B2-REF', 'B3-REF', 'gg_13_8_otus', 'F1-REF', 'F2-REF', 'F3-REF', 'unite_20.11.2016_clean_fullITS']
# Set level
mock_results = mock_results[mock_results['Level'] == level]
nt_results = nt_results[nt_results['level'] == level]
# Filter mock results to use correct datasets
mock_results = mock_results[(mock_results[reference].isin(ref_ids)) &
(mock_results[method].isin(method_ids)) &
(mock_results[dataset].isin(dataset_ids))]
# Filter nt and cv results to use correct datasets
cv_results = cv_results[cv_results[dataset].isin(ref_ids)]
nt_results = nt_results[nt_results[dataset].isin(ref_ids)]
# modify parameter names so all evaluations use same conventions
mock_results[params] = mock_results[params].str.replace('prior:char:8192','')
mock_results[params] = mock_results[params].str.replace('char:8192:','')
# extract per-level results from cv_results
for m in ['Precision', 'Recall', 'F-measure']:
level_results = []
for v in cv_results[m]:
v = v.strip('[]').split(', ')
level_results.append(float(v[level]))
cv_results[m] = level_results
# join all eval dataframes into one
mock_mean = mock_results.groupby([method, params]).mean()
cv_mean = cv_results.groupby([method, params]).mean()
nt_mean = nt_results.groupby([method, params]).mean()
dat = ['F-measure', 'Precision', 'Recall']
evals_combined = pd.concat([mock_mean[dat], cv_mean[dat], nt_mean[dat]], axis=1, join='inner')
evals_combined.columns = ['Mock', 'MP', 'MR', 'Cross-validated', 'CP', 'CR', 'Novel taxa', 'NP', 'NR']
return evals_combined
def plot_3d(evals_combined, x='Mock', y='Cross-validated', z='Novel taxa'):
fig = plt.figure()
ax = fig.gca(projection='3d')
ax.set_facecolor('white')
for m in evals_combined.index.levels[0]:
evals_subset = evals_combined.loc[m]
ax.scatter(xs=evals_subset[x],
ys=evals_subset[y],
zs=evals_subset[z],
c=color_palette[m])
ax.set_xlabel(x)
ax.set_ylabel(y)
ax.set_zlabel(z)
plt.show()
return ax
def set_threshold(df, mock_threshold, cv_threshold, nt_threshold, x='Mock', y='Cross-validated', z='Novel taxa'):
mock_threshold = df[x] >= mock_threshold
cv_threshold = df[y] >= cv_threshold
nt_threshold = df[z] >= nt_threshold
df = df[mock_threshold & cv_threshold & nt_threshold]
return df
In [167]:
evals_bacteria_5 = combined_evaluation(
mock_results, cv_results, nt_results, params,
metric, method, reference, dataset, "bacteria", 5)
ax = plot_3d(evals_bacteria_5)
ax.get_figure().savefig(join(outdir, "3way-eval-fmeasure-bacteria-genus.pdf"), facecolor="white")
ax.get_figure().savefig(join(outdir, "3way-eval-fmeasure-bacteria-genus.png"), facecolor="white")
In [169]:
# find top-performing methods across all evals
evals_bacteria_5['sum'] = evals_bacteria_5['Mock'] + evals_bacteria_5['Cross-validated'] + evals_bacteria_5['Novel taxa']
evals_bacteria_5.sort_values('sum', ascending=False)[:10]
Out[169]:
In [171]:
evals_bacteria = combined_evaluation(
mock_results, cv_results, nt_results, params,
metric, method, reference, dataset, "bacteria", level)
ax = plot_3d(evals_bacteria)
ax.get_figure().savefig(join(outdir, "3way-eval-fmeasure-bacteria.pdf"), facecolor="white")
ax.get_figure().savefig(join(outdir, "3way-eval-fmeasure-bacteria.png"), facecolor="white")
In [172]:
# find top-performing methods across all evals
evals_bacteria['sum'] = evals_bacteria['Mock'] + evals_bacteria['Cross-validated'] + evals_bacteria['Novel taxa']
evals_bacteria.sort_values('sum', ascending=False)[:10]
Out[172]:
In [174]:
set_threshold(evals_bacteria, 0.7, 0.8, 0.15)
Out[174]:
In [175]:
set_threshold(evals_bacteria, 0.4, 0.8, 0.2)
Out[175]:
In [176]:
set_threshold(evals_bacteria, 0.9, 0.75, 0.05)
Out[176]:
In [177]:
a = mock_results[(mock_results['Level'] == 6) & (mock_results['Reference'] == 'gg_13_8_otus')]
a = a.groupby(['Method', 'Parameters']).mean()
a = a.sort_values('F-measure', ascending=False)
a[:20]
Out[177]:
In [180]:
ax = plot_3d(evals_bacteria, 'MP', 'CP', 'NP')
In [207]:
set_threshold(evals_bacteria_precision, 0.95, 0.95, 0.25, 'MP', 'CP', 'NP')
Out[207]:
In [183]:
ax = plot_3d(evals_bacteria, 'MR', 'CR', 'NR')
In [185]:
set_threshold(evals_bacteria, 0.7, 0.75, 0.04, 'MR', 'CR', 'NR')
Out[185]:
In [186]:
evals_fungi = combined_evaluation(
mock_results, cv_results, nt_results, params,
metric, method, reference, dataset, "fungi", level)
ax = plot_3d(evals_fungi)
ax.get_figure().savefig(join(outdir, "3way-eval-fmeasure-fungi.pdf"), facecolor="white")
ax.get_figure().savefig(join(outdir, "3way-eval-fmeasure-fungi.png"), facecolor="white")
In [187]:
# find top-performing methods across all evals
evals_fungi['sum'] = evals_fungi['Mock'] + evals_fungi['Cross-validated'] + evals_fungi['Novel taxa']
evals_fungi.sort_values('sum', ascending=False)[:10]
Out[187]:
In [188]:
set_threshold(evals_fungi, 0.85, 0.45, 0.37)
Out[188]:
In [198]:
set_threshold(evals_fungi, 0.85, 0.45, 0.4)
Out[198]:
In [189]:
evals_fungi_5 = combined_evaluation(
mock_results, cv_results, nt_results, params,
metric, method, reference, dataset, "fungi", 5)
ax = plot_3d(evals_fungi_5)
ax.get_figure().savefig(join(outdir, "3way-eval-fmeasure-fungi-genus.pdf"), facecolor="white")
ax.get_figure().savefig(join(outdir, "3way-eval-fmeasure-fungi-genus.png"), facecolor="white")
In [190]:
# find top-performing methods across all evals
evals_fungi_5['sum'] = evals_fungi_5['Mock'] + evals_fungi_5['Cross-validated'] + evals_fungi_5['Novel taxa']
evals_fungi_5.sort_values('sum', ascending=False)[:10]
Out[190]:
In [191]:
ax = plot_3d(evals_fungi, 'MP', 'CP', 'NP')
In [205]:
set_threshold(evals_fungi, 0.92, 0.6, 0.3, 'MP', 'CP', 'NP')
Out[205]:
In [193]:
ax = plot_3d(evals_fungi, 'MR', 'CR', 'NR')
In [194]:
set_threshold(evals_fungi, 0.9, 0.4, 0.3, 'MR', 'CR', 'NR')
Out[194]:
In [26]:
p = plt.figure(figsize=(24, 8))
c = 0
for d in ["bacteria", "fungi"]:
if d == "fungi":
c = 5
for l in range(2,7):
evals_combined = combined_evaluation(
mock_results, cv_results, nt_results, params,
metric, method, reference, dataset, d, l)
#p = plt.subplot(5, 2, l-1)
#ax = p.gca(projection='3d')
ax = p.add_subplot(2, 5, l-1+c, projection='3d')
for m in evals_combined.index.levels[0]:
evals_subset = evals_combined.loc[m]
ax.scatter(xs=evals_subset['Mock'],
ys=evals_subset['Cross-validated'],
zs=evals_subset['Novel taxa'],
c=color_palette[m])
ax.set_xlabel('Mock')
ax.set_ylabel('Cross-validated')
ax.set_zlabel('Novel taxa')
plt.show()
In [40]:
p = plt.figure(figsize=(24, 12))
c = 0
p.set_facecolor('white')
for d in ["bacteria", "fungi"]:
if d == "fungi":
c = 3
for metric in ['F-measure', 'Precision', 'Recall']:
evals_combined = combined_evaluation(
mock_results, cv_results, nt_results, params,
metric, method, reference, dataset, d, 6)
c+=1
ax = p.add_subplot(2, 3, c, projection='3d')
ax.set_facecolor('white')
for m in evals_combined.index.levels[0]:
evals_subset = evals_combined.loc[m]
ax.scatter(xs=evals_subset['Mock'],
ys=evals_subset['Cross-validated'],
zs=evals_subset['Novel taxa'],
c=color_palette[m])
ax.set_xlabel('Mock')
ax.set_ylabel('Cross-validated')
ax.set_zlabel('Novel taxa')
plt.show()
In [41]:
p.savefig(join(outdir, "3way-eval.png"), facecolor="white")
p.savefig(join(outdir, "3way-eval.pdf"), facecolor="white")
In [ ]: