Experiment: type-I, type-II errors as d increases


In [ ]:
%load_ext autoreload
%autoreload 2
%matplotlib inline
#%config InlineBackend.figure_format = 'svg'
#%config InlineBackend.figure_format = 'pdf'
import freqopttest.util as util
import freqopttest.data as data
import freqopttest.ex.exglobal as exglo
import freqopttest.kernel as kernel
import freqopttest.tst as tst
import freqopttest.glo as glo
import freqopttest.plot as plot
import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import scipy.stats as stats
import sys

In [ ]:
def load_plot_vs_d(fname, h1_true, show_legend=True):
    func_xvalues = lambda results: results['dimensions']
    def func_title(results):
        repeats, _, n_methods = results['test_results'].shape
        alpha = results['alpha']
        test_size = (1.0 - results['tr_proportion'])*results['sample_size']
        title = '%s. %d trials. test size: %d. $\\alpha$ = %.2g.'%\
            ( results['prob_label'], repeats, test_size, alpha)
        return title
    results = plot.plot_prob_stat_above_thresh(2, fname, h1_true, func_xvalues, 'Dimension', func_title)
    
    plt.title('')
    plt.gca().legend().set_visible(show_legend)
        
    return results



def load_runtime_vs_d(fname, h1_true=True, xlabel='Dimension', 
                      show_legend=True, xscale='linear', yscale='log'):
    func_xvalues = lambda agg_results: agg_results['dimensions']
    ex = 2
    def func_title(agg_results):
        repeats, _, n_methods = agg_results['test_results'].shape
        alpha = agg_results['alpha']
        title = '%s. %d trials. $\\alpha$ = %.2g.'%\
            ( agg_results['prob_label'], repeats, alpha)
        return title
    
    #plt.figure(figsize=(10,6))
    
    results = plot.plot_runtime(ex, fname,  
                                func_xvalues, xlabel=xlabel, func_title=func_title)
    
    plt.title('')
    plt.gca().legend(loc='best').set_visible(show_legend)
    #plt.grid(True)
    if xscale is not None:
        plt.xscale(xscale)
    if yscale is not None:
        plt.yscale(yscale)
    
    xvalues = func_xvalues(results)
    plt.xticks(xvalues)
    plt.autoscale(tight=True)
    
    return results

In [ ]:
# font options
font = {
    #'family' : 'normal',
    #'weight' : 'bold',
    'size'   : 18
}

plt.rc('font', **font)
plt.rc('lines', linewidth=2)
matplotlib.rcParams['pdf.fonttype'] = 42
matplotlib.rcParams['ps.fonttype'] = 42

In [ ]:
# H0 true. Same Gauss. 
sg_fname = 'ex2-sg-me6_J5_rs200_n20000_dmi5_dma1500_a0.010_trp0.50.p'
#sg_fname = 'ex2-sg-me6_J5_rs50_n20000_dmi5_dma1500_a0.010_trp0.50.p'
sg_results = load_plot_vs_d(sg_fname, False, show_legend=False)
plt.ylim([0, 0.025])
plt.savefig(sg_fname.replace('.p', '.pdf', 1), bbox_inches='tight')

In [ ]:
load_runtime_vs_d(sg_fname, xscale='linear', yscale='log', show_legend=False);
#plt.legend(bbox_to_anchor=(1.7, 1))
plt.savefig(sg_fname.replace('.p', '', 1) + '_time.pdf', bbox_inches='tight')

In [ ]:
# H1 true. Gaussian. mean diff
gmd_fname = 'ex2-gmd-me6_J5_rs200_n20000_dmi5_dma1500_a0.010_trp0.50.p'
#gmd_fname = 'ex2-gmd-me6_J5_rs50_n20000_dmi5_dma1500_a0.010_trp0.50.p'
gmd_results = load_plot_vs_d(gmd_fname, True, show_legend=False)
plt.ylim([0, 1.03])
plt.savefig(gmd_fname.replace('.p', '.pdf', 1), bbox_inches='tight')

In [ ]:
load_runtime_vs_d(gmd_fname, xscale='linear', yscale='log', show_legend=False);
#plt.legend(bbox_to_anchor=(1.7, 1))
plt.savefig(gmd_fname.replace('.p', '', 1) + '_time.pdf', bbox_inches='tight')

In [ ]:
# H1 true. Gaussian. variance diff
gvd_fname = 'ex2-gvd-me6_J5_rs200_n20000_dmi5_dma500_a0.010_trp0.50.p'
gvd_results = load_plot_vs_d(gvd_fname, True, show_legend=False)
plt.legend(bbox_to_anchor=(1.7, 0.95))
plt.savefig(gvd_fname.replace('.p', '.pdf', 1),  bbox_inches='tight')

In [ ]:
load_runtime_vs_d(gvd_fname, xscale='linear', yscale='log', show_legend=False);
plt.legend(bbox_to_anchor=(1.7, 1))
plt.savefig(gvd_fname.replace('.p', '', 1) + '_time.pdf', bbox_inches='tight')

In [ ]: