A notebook to process experimental results of ex2_prob_params.py. p(reject) as problem parameters are varied.


In [ ]:
%load_ext autoreload
%autoreload 2
%matplotlib inline
#%config InlineBackend.figure_format = 'svg'
#%config InlineBackend.figure_format = 'pdf'

import numpy as np

import matplotlib
import matplotlib.pyplot as plt
import kgof.data as data
import kgof.glo as glo
import kgof.goftest as gof
import kgof.kernel as kernel
import kgof.plot as plot
import kgof.util as util

import scipy.stats as stats

In [ ]:
import kgof.plot
kgof.plot.set_default_matplotlib_options()

In [ ]:
def load_plot_vs_params(fname, xlabel='Problem parameter', show_legend=True):
    func_xvalues = lambda agg_results: agg_results['prob_params']
    ex = 2
    def func_title(agg_results):
        repeats, _, n_methods = agg_results['job_results'].shape
        alpha = agg_results['alpha']
        test_size = (1.0 - agg_results['tr_proportion'])*agg_results['sample_size']
        title = '%s. %d trials. test size: %d. $\\alpha$ = %.2g.'%\
            ( agg_results['prob_label'], repeats, test_size, alpha)
        return title
    #plt.figure(figsize=(10,5))
    results = plot.plot_prob_reject(
        ex, fname, func_xvalues, xlabel, func_title=func_title)
    
    plt.title('')
    plt.gca().legend(loc='best').set_visible(show_legend)
    if show_legend:
        plt.legend(bbox_to_anchor=(1.80, 1.08))
    
    plt.grid(False)
        
    return results


def load_runtime_vs_params(fname, xlabel='Problem parameter', 
                      show_legend=True, xscale='linear', yscale='linear'):
    func_xvalues = lambda agg_results: agg_results['prob_params']
    ex = 2
    def func_title(agg_results):
        repeats, _, n_methods = agg_results['job_results'].shape
        alpha = agg_results['alpha']
        title = '%s. %d trials. $\\alpha$ = %.2g.'%\
            ( agg_results['prob_label'], repeats, alpha)
        return title
    
    #plt.figure(figsize=(10,6))
    
    results = plot.plot_runtime(ex, fname,  
                                func_xvalues, xlabel=xlabel, func_title=func_title)
    
    plt.title('')
    plt.gca().legend(loc='best').set_visible(show_legend)
    if show_legend:
        plt.legend(bbox_to_anchor=(1.80, 1.05))
    
    plt.grid(False)
    if xscale is not None:
        plt.xscale(xscale)
    if yscale is not None:
        plt.yscale(yscale)
        
    return results

In [ ]:
# # Gaussian mean difference. Fix dimension. Vary the mean
# #gmd_fname = 'ex2-gmd_d10_ms-me5_n1000_rs100_pmi0.000_pma0.600_a0.050_trp0.50.p'
# gmd_fname = 'ex2-gmd_d10_ms-me4_n2000_rs50_pmi0.000_pma0.060_a0.050_trp0.50.p'
# gmd_results = load_plot_vs_params(gmd_fname, xlabel='$m$', show_legend=True)
# #plt.ylim([0.03, 0.1])
# #plt.savefig(bsg_fname.replace('.p', '.pdf', 1), bbox_inches='tight')
$$p(x) = \mathcal{N}(0, I) \\ q(x) = \mathcal{N}((m,0,\ldots), I)$$

In [ ]:
# # Gaussian increasing variance. Variance below 1.
# gvsub1_d1_fname = 'ex2-gvsub1_d1_vs-me8_n1000_rs100_pmi0.100_pma0.700_a0.050_trp0.50.p'
# gvsub1_d1_results = load_plot_vs_params(gvsub1_d1_fname, xlabel='$v$')
# plt.title('d=1')
# # plt.ylim([0.02, 0.08])
# # plt.xlim([0, 4])
# #plt.legend(bbox_to_anchor=(1.70, 1.05))
# #plt.savefig(gsign_fname.replace('.p', '.pdf', 1), bbox_inches='tight')

In [ ]:
# # Gaussian increasing variance
# gvinc_d5_fname = 'ex2-gvinc_d5-me8_n1000_rs100_pmi1.000_pma2.500_a0.050_trp0.50.p'
# gvinc_d5_results = load_plot_vs_params(gvinc_d5_fname, xlabel='$v$', 
#     show_legend=True)
# plt.title('d=5')
# # plt.ylim([0.02, 0.08])
# # plt.xlim([0, 4])
# #plt.legend(bbox_to_anchor=(1.70, 1.05))
# #plt.savefig(gsign_fname.replace('.p', '.pdf', 1), bbox_inches='tight')
$$p(x)=\mathcal{N}(0, I) \\ q(x)=\mathcal{N}(0, vI)$$

In [ ]:
# # Gaussian variance diffenece (GVD)
# gvd_fname = 'ex2-gvd-me4_n1000_rs100_pmi1.000_pma15.000_a0.050_trp0.50.p'
# # gvd_fname = 'ex2-gvd-me4_n1000_rs50_pmi1.000_pma15.000_a0.050_trp0.80.p'
# gvd_results = load_plot_vs_params(gvd_fname, xlabel='$d$', show_legend=True)
# plt.figure()
# load_runtime_vs_params(gvd_fname);
$$p(x)=\mathcal{N}(0, I) \\ q(x)=\mathcal{N}(0, \mathrm{diag}(2,1,1,\ldots))$$

In [ ]:
# Gauss-Bernoulli RBM
# gb_rbm_fname = 'ex2-gbrbm_dx50_dh10-me4_n1000_rs200_pmi0.000_pma0.001_a0.050_trp0.20.p'
# gb_rbm_fname = 'ex2-gbrbm_dx50_dh10-me4_n1000_rs200_pmi0.000_pma0.000_a0.050_trp0.20.p'
# gb_rbm_fname = 'ex2-gbrbm_dx50_dh10-me6_n1000_rs300_pmi0.000_pma0.001_a0.050_trp0.20.p'
gb_rbm_fname = 'ex2-gbrbm_dx50_dh10-me6_n1000_rs200_pmi0_pma0.06_a0.050_trp0.20.p'
gb_rbm_results = load_plot_vs_params(gb_rbm_fname, xlabel='Perturbation SD $\sigma_{per}$', 
                                     show_legend=False)
plt.savefig(gb_rbm_fname.replace('.p', '.pdf', 1), bbox_inches='tight')
# plt.xlim([-0.1, -0.2])

In [ ]:
load_runtime_vs_params(gb_rbm_fname,  xlabel='Perturbation SD $\sigma_{per}$', yscale='linear', show_legend=False);
plt.savefig(gb_rbm_fname.replace('.p', '_time.pdf', 1), bbox_inches='tight')

In [ ]:
# gbrbm_highd_fname = 'ex2-gbrbm_dx50_dh40-me6_n1000_rs200_pmi0_pma0.06_a0.050_trp0.20.p'
# gbrbm_highd_fname = 'ex2-gbrbm_dx50_dh40-me2_n1000_rs200_pmi0_pma0.06_a0.050_trp0.20.p'
gbrbm_highd_fname = 'ex2-gbrbm_dx50_dh40-me1_n1000_rs200_pmi0_pma0.06_a0.050_trp0.20.p'
gbrbm_highd_results = load_plot_vs_params(
    gbrbm_highd_fname, 
#     xlabel='Perturbation SD $\sigma_{per}$', 
    xlabel='Perturbation noise', 
                                          show_legend=False)
plt.xticks([0, 0.02, 0.04, 0.06])
plt.yticks([0, 0.5, 1])
plt.ylim([0, 1.05])
plt.ylabel('P(detect difference)', fontsize=26)
plt.box(True)
plt.savefig(gbrbm_highd_fname.replace('.p', '.pdf', 1), bbox_inches='tight')

In [ ]:
load_runtime_vs_params(gbrbm_highd_fname,  xlabel='Perturbation SD $\sigma_{per}$', 
                       yscale='linear', show_legend=False);
plt.savefig(gbrbm_highd_fname.replace('.p', '_time.pdf', 1), bbox_inches='tight')

In [ ]:
## p: Gaussian, q: Laplace. Vary d
# glaplace_fname = 'ex2-glaplace-me4_n1000_rs100_pmi1.000_pma15.000_a0.050_trp0.50.p'
# glaplace_fname = 'ex2-glaplace-me4_n1000_rs200_pmi1.000_pma15.000_a0.050_trp0.20.p'
# glaplace_fname = 'ex2-glaplace-me5_n1000_rs400_pmi1.000_pma15.000_a0.050_trp0.20.p'
glaplace_fname = 'ex2-glaplace-me6_n1000_rs200_pmi1_pma15_a0.050_trp0.20.p'
glaplace_results = load_plot_vs_params(glaplace_fname, xlabel='dimension $d$', show_legend=False)
plt.savefig(glaplace_fname.replace('.p', '.pdf', 1), bbox_inches='tight')

In [ ]:
load_runtime_vs_params(glaplace_fname, xlabel='dimension $d$', show_legend=False, yscale='linear');
plt.savefig(glaplace_fname.replace('.p', '_time.pdf', 1), bbox_inches='tight')
$$p(x)=\mathcal{N}(0, 1) \\ q(x)=\mathrm{Laplace}(0, 1/\sqrt{2})$$

q has the same unit variance as p.


In [ ]:


In [ ]:


In [ ]: