Now that I've streamlined the MCMC process, I am going to submit multiple chains simultaneously. This notebook will make multiple, similar config files, for broad comparison.

This may be rolled into pearce as a helper function, I haven't decided.

Update from the older SLAC notebook, which is now outdated

For rmin 0, 0.5, 1.0:

For no ab, HSAB and CorrAB emu:

    Vpeak sham
    Mpeak sham
    HOD
    HSAB HOD



In [1]:

    
import yaml
import copy
from os import path
import numpy as np



In [2]:

    
emu_path = '/u/ki/swmclau2/des/'
#emu_path = '~/scratch'



In [3]:

    
cov_path = '/u/ki/swmclau2/Git/pearce/bin/'
# cov_path = '~/Git/pearce/bin'



In [4]:

    
orig_cfg_fname = path.join(cov_path, 'mcmc/nh_gg_sham_hsab_mcmc_config.yaml')
with open(orig_cfg_fname, 'r') as yamlfile:
    orig_cfg = yaml.load(yamlfile)



In [5]:

    
orig_cfg









    Out[5]:





{'chain': {'fixed_params': {},
  'nburn': 0,
  'nsteps': 10000,
  'nwalkers': 500,
  'seed': 0},
 'data': {'cov': {'emu_cov_fname': '/home/users/swmclau2/Git/pearce/bin/covmat/xi_gg_nh_emu_cov.npy',
   'meas_cov_fname': '/home/users/swmclau2/Git/pearce/notebooks/meas_cov_testboxes_gg.npy'},
  'obs': {'mean': False,
   'obs': 'xi',
   'rbins': [0.07943282,
    0.11220185,
    0.15848932,
    0.22387211,
    0.31622777,
    0.44668359,
    0.63095734,
    0.89125094,
    1.25892541,
    1.77827941,
    2.51188643,
    3.54813389,
    5.01187234,
    7.07945784,
    10.0,
    14.12537545,
    19.95262315,
    28.18382931,
    39.81071706]},
  'sim': {'gal_type': 'SHAM',
   'halo_property': 'halo_vpeak',
   'min_ptcl': 100,
   'nd': '5e-4',
   'scale_factor': 1.001,
   'scatter': 0.2,
   'sim_hps': {'system': 'sherlock'},
   'simname': 'ds_14_b_sub'},
  'true_cov_fname': ['/home/users/swmclau2/Git/pearce/bin/shams/xigg_cov_mcmc.npy'],
  'true_data_fname': ['/home/users/swmclau2/Git/pearce/bin/shams/ds14b_sub_xi_gg.npy']},
 'emu': {'emu_hps': {},
  'emu_type': 'NashvilleHot',
  'fixed_params': {'z': 0.0},
  'training_file': '/home/users/swmclau2/scratch/xi_gg_abzheng07/PearceXiggHSABCosmo.hdf5'},
 'fname': '/scratch/users/swmclau2/PearceMCMC/pearce_mcmc_nh_gg_sham_hsab.hdf5'}

#this will enable easier string formatting sbatch_template = """#!/bin/bash #SBATCH --job-name={jobname} #SBATCH --time=24:00:00 #SBATCH -p kipac,iric #SBATCH -o /home/users/swmclau2/Git/pearce/bin/mcmc/config/{jobname}.out #SBATCH --nodes=1 #SBATCH --ntasks=1 #SBATCH --cpus-per-task=16 ###SBATCH --exclusive module load python/2.7.13 module load py-scipystack module load hdf5/1.10.0p1 module load py-numpy python /home/users/swmclau2/Git/pearce/pearce/inference/initialize_mcmc.py {jobname}.yaml python /home/users/swmclau2/Git/pearce/pearce/inference/run_mcmc.py {jobname}.yaml """

bsub_template="""#BSUB -q medium #BSUB -W 8:00 #BSUB -J {jobname} #BSUB -oo /u/ki/swmclau2/Git/pearce/bin/mcmc/config/{jobname}.out #BSUB -n 8 #BSUB -R "span[ptile=8]" python /u/ki/swmclau2/Git/pearce/pearce/inference/initialize_mcmc.py {jobname}.yaml python /u/ki/swmclau2/Git/pearce/pearce/inference/run_mcmc.py {jobname}.yaml """



In [6]:

    
bsub_restart_template="""#BSUB -q medium 
#BSUB -W 8:00
#BSUB -J {jobname}
#BSUB -oo /u/ki/swmclau2/Git/pearce/bin/mcmc/config/{jobname}.out    
#BSUB -n 8 
#BSUB -R "span[ptile=8]"

#python /u/ki/swmclau2/Git/pearce/pearce/inference/initialize_mcmc.py {jobname}.yaml
python /u/ki/swmclau2/Git/pearce/pearce/inference/run_mcmc.py {jobname}.yaml --restart
"""



In [7]:

    
r_bins = np.logspace(-1, 1.6, 19)



In [8]:

    
emu_names = ['HOD','HSAB','CAB','CorrAB']
emu_fnames = [[path.join(emu_path, 'Aemulators/wp_zheng07/PearceWpCosmo.hdf5'), path.join(emu_path, 'Aemulators/ds_zheng07/PearceDsCosmo.hdf5')],\
              [path.join(emu_path, 'Aemulators/wp_hsab/PearceWpHSABCosmo.hdf5'), path.join(emu_path, 'Aemulators/ds_hsab/PearceDsHSABCosmo.hdf5')],
              [path.join(emu_path, 'Aemulators/wp_cab/PearceWpCABCosmo.hdf5'), path.join(emu_path,'Aemulators/ds_cab/PearceDsCABCosmo.hdf5')],
              [path.join(emu_path, 'Aemulators/wp_corrab/PearceWpCorrABCosmo.hdf5'), path.join(emu_path, 'Aemulators/ds_corrab/PearceDsCorrABCosmo.hdf5')]]



In [9]:

    
meas_cov_fname = path.join(cov_path, 'covmat/wp_ds_full_covmat.npy')

emu_cov_fnames = [[path.join(cov_path, 'optimization/wp_hod_emu_cov_lpw.npy'), 
                   path.join(cov_path, 'optimization/ds_hod_emu_cov_lpw.npy')],
                  [path.join(cov_path, 'optimization/wp_hsab_emu_cov_lpw.npy'), 
                   path.join(cov_path, 'optimization/ds_hsab_emu_cov_lpw.npy')],
                 [path.join(cov_path, 'optimization/wp_cab_emu_cov_lpw.npy'), 
                   path.join(cov_path, 'optimization/ds_cab_emu_cov_lpw.npy')],
                 [path.join(cov_path, 'optimization/wp_corrab_emu_cov_lpw.npy'),
                  path.join(cov_path, 'optimization/ds_corrab_emu_cov_lpw.npy')]]



In [10]:

    
n_walkers = 250
n_steps = 100000

Vpeak SHAM



In [11]:

    
tmp_cfg = copy.deepcopy(orig_cfg)

directory = path.join(cov_path, "mcmc/config/")
output_dir = path.join(emu_path, "PearceMCMC/")
jobname_template = "VpeakSHAM_wp_ds_rmin_{rmin}_{emu_name}"

for rmin in [None, 0.5, 1.0, 2.0]:
    for emu_fname, emu_name, emu_cov in zip(emu_fnames, emu_names, emu_cov_fnames):
        
        if rmin is not None:
            tmp_cfg['emu']['fixed_params'] = {'z': 0.0, 'rmin':rmin}
            
        tmp_cfg['emu']['training_file'] = emu_fname
        tmp_cfg['emu']['emu_type'] = ['LemonPepperWet' for i in xrange(len(emu_fname))]
        tmp_cfg['emu']['emu_cov_fname'] = emu_cov
        
        tmp_cfg['data']['true_data_fname']= [path.join(cov_path, 'mock_measurements/SHAMmock_wp.npy'),
                                             path.join(cov_path, 'mock_measurements/SHAMmock_ds.npy')]
        tmp_cfg['data']['true_cov_fname'] = meas_cov_fname
        
        tmp_cfg['data']['obs']['obs'] = ['wp','ds']
        tmp_cfg['data']['obs']['rbins'] = list(r_bins)
        

        tmp_cfg['chain']['nsteps'] = n_steps
        tmp_cfg['chain']['nwalkers'] = n_walkers
        tmp_cfg['chain']['mcmc_type'] = 'normal'

        
        tmp_cfg['data']['sim']['sim_hps']['system'] = 'ki-ls'
        tmp_cfg['data']['cov']['emu_cov_fname'] = tmp_cfg['emu']['emu_cov_fname'] 
        tmp_cfg['data']['cov']['meas_cov_fname'] = tmp_cfg['data']['true_cov_fname']
        
        jobname = jobname_template.format(rmin=rmin, emu_name=emu_name)
        tmp_cfg['fname'] = path.join(output_dir, jobname+'.hdf5')

        with open(path.join(directory, jobname +'.yaml'), 'w') as f:
            yaml.dump(tmp_cfg, f)

        #with open(path.join(directory, jobname + '.sbatch'), 'w') as f:
        #    f.write(sbatch_template.format(jobname=jobname))
        with open(path.join(directory, jobname + '.bsub'), 'w') as f:
            f.write(bsub_template.format(jobname=jobname))
            #f.write(bsub_restart_template.format(jobname=jobname))

NFWized SHAM



In [13]:

    
tmp_cfg = copy.deepcopy(orig_cfg)
directory = path.join(cov_path, "mcmc/config/")
output_dir = path.join(emu_path, "PearceMCMC/")
jobname_template = "NFWizedSHAM_wp_ds_rmin_{rmin}_{emu_name}"

for rmin in [None, 0.5, 1.0, 2.0]:
    for emu_fname, emu_name, emu_cov in zip(emu_fnames, emu_names, emu_cov_fnames):
        
        if rmin is not None:
            tmp_cfg['emu']['fixed_params'] = {'z': 0.0, 'rmin':rmin}
            
        tmp_cfg['emu']['training_file'] = emu_fname
        tmp_cfg['emu']['emu_type'] = ['LemonPepperWet' for i in xrange(len(emu_fname))]
        tmp_cfg['emu']['emu_cov_fname'] = emu_cov
        
        tmp_cfg['data']['true_data_fname']= [path.join(cov_path,'mock_measurements/NFWIZED_SHAMmock_wp.npy'),
                                             path.join(cov_path,'mock_measurements/NFWIZED_SHAMmock_ds.npy')]
        tmp_cfg['data']['true_cov_fname'] = meas_cov_fname
        
        tmp_cfg['data']['obs']['obs'] = ['wp','ds']
        tmp_cfg['data']['obs']['rbins'] = list(r_bins)
        

        tmp_cfg['chain']['nsteps'] = n_steps
        tmp_cfg['chain']['nwalkers'] = n_walkers
        tmp_cfg['chain']['mcmc_type'] = 'normal'

        
        tmp_cfg['data']['sim']['sim_hps']['system'] = 'ki-ls'
        tmp_cfg['data']['cov']['emu_cov_fname'] = tmp_cfg['emu']['emu_cov_fname'] 
        tmp_cfg['data']['cov']['meas_cov_fname'] = tmp_cfg['data']['true_cov_fname']
        
        jobname = jobname_template.format(rmin=rmin, emu_name=emu_name)
        tmp_cfg['fname'] = path.join(output_dir, jobname+'.hdf5')

        with open(path.join(directory, jobname +'.yaml'), 'w') as f:
            yaml.dump(tmp_cfg, f)

        with open(path.join(directory, jobname + '.bsub'), 'w') as f:
            f.write(bsub_template.format(jobname=jobname))
            #f.write(bsub_restart_template.format(jobname=jobname))



In [14]:

    
h = 0.6777

cosmo_pnames = ['ombh2', 'omch2', 'w0', 'ns', 'H0', 'Neff', 'sigma8']
cosmo_true_vals = np.array([0.048206*h**2, 0.307115*h**2,\
                            -1, 0.9667, \
                            h*100, 3.046, 0.8228])

# fixed cosmology tmp_cfg = copy.deepcopy(orig_cfg) directory = path.join(cov_path, "mcmc/config/") output_dir = path.join(emu_path, "PearceMCMC/") jobname_template = "NFWizedSHAM_fixed_cosmo_wp_ds_rmin_{rmin}_{emu_name}" for rmin in [None, 0.5, 1.0, 2.0]: for emu_fname, emu_name, emu_cov in zip(emu_fnames, emu_names, emu_cov_fnames): if rmin is not None: tmp_cfg['emu']['fixed_params'] = {'z': 0.0, 'rmin':rmin} tmp_cfg['emu']['training_file'] = emu_fname tmp_cfg['emu']['emu_type'] = ['LemonPepperWet' for i in xrange(len(emu_fname))] tmp_cfg['emu']['emu_cov_fname'] = emu_cov tmp_cfg['data']['true_data_fname']= [path.join(cov_path,'mock_measurements/NFWIZED_SHAMmock_wp.npy'), path.join(cov_path,'mock_measurements/NFWIZED_SHAMmock_ds.npy')] tmp_cfg['data']['true_cov_fname'] = meas_cov_fname tmp_cfg['data']['obs']['obs'] = ['wp','ds'] tmp_cfg['data']['obs']['rbins'] = list(r_bins) tmp_cfg['chain']['nsteps'] = n_steps tmp_cfg['chain']['nwalkers'] = n_walkers tmp_cfg['chain']['mcmc_type'] = 'normal' tmp_cfg['data']['sim']['sim_hps']['system'] = 'ki-ls' tmp_cfg['data']['cov']['emu_cov_fname'] = tmp_cfg['emu']['emu_cov_fname'] tmp_cfg['data']['cov']['meas_cov_fname'] = tmp_cfg['data']['true_cov_fname'] tmp_cfg['chain']['fixed_params'] = 'cosmo' tmp_cfg['sim']= {'gal_type': 'SHAM', 'nd': '5e-4', 'scale_factor': 1.0, 'min_ptcl': 100, 'cosmo_params': dict(zip(cosmo_pnames, cosmo_true_vals)), 'sim_hps': {}, 'simname': 'mdpl2'} # TODO i shouldnt have to specify this this way tmp_cfg['data']['sim'] = tmp_cfg['sim'] jobname = jobname_template.format(rmin=rmin, emu_name=emu_name) tmp_cfg['fname'] = path.join(output_dir, jobname+'.hdf5') with open(path.join(directory, jobname +'.yaml'), 'w') as f: yaml.dump(tmp_cfg, f) with open(path.join(directory, jobname + '.bsub'), 'w') as f: f.write(bsub_template.format(jobname=jobname)) #f.write(bsub_restart_template.format(jobname=jobname))

Shuffled SHAM



In [16]:

    
tmp_cfg = copy.deepcopy(orig_cfg)

directory = path.join(cov_path, "mcmc/config/")
output_dir = path.join(emu_path, "PearceMCMC/")
jobname_template = "ShuffledSHAM_wp_ds_rmin_{rmin}_{emu_name}"

for rmin in [None, 0.5, 1.0, 2.0]:
    for emu_fname, emu_name, emu_cov in zip(emu_fnames, emu_names, emu_cov_fnames):
        
        if rmin is not None:
            tmp_cfg['emu']['fixed_params'] = {'z': 0.0, 'rmin':rmin}
            
        tmp_cfg['emu']['training_file'] = emu_fname
        tmp_cfg['emu']['emu_type'] = ['LemonPepperWet' for i in xrange(len(emu_fname))]
        tmp_cfg['emu']['emu_cov_fname'] = emu_cov
        
        tmp_cfg['data']['true_data_fname']= [path.join(cov_path, 'mock_measurements/SHUFFLED_SHAMmock_wp.npy'),
                                             path.join(cov_path, 'mock_measurements/SHUFFLED_SHAMmock_ds.npy')]
        tmp_cfg['data']['true_cov_fname'] = meas_cov_fname
        
        tmp_cfg['data']['obs']['obs'] = ['wp','ds']
        tmp_cfg['data']['obs']['rbins'] = list(r_bins)
        

        tmp_cfg['chain']['nsteps'] = n_steps
        tmp_cfg['chain']['nwalkers'] = n_walkers
        tmp_cfg['chain']['mcmc_type'] = 'normal'

        
        tmp_cfg['data']['sim']['sim_hps']['system'] = 'ki-ls'
        tmp_cfg['data']['cov']['emu_cov_fname'] = tmp_cfg['emu']['emu_cov_fname'] 
        tmp_cfg['data']['cov']['meas_cov_fname'] = tmp_cfg['data']['true_cov_fname']
        
        jobname = jobname_template.format(rmin=rmin, emu_name=emu_name)
        tmp_cfg['fname'] = path.join(output_dir, jobname+'.hdf5')

        with open(path.join(directory, jobname +'.yaml'), 'w') as f:
            yaml.dump(tmp_cfg, f)

        #with open(path.join(directory, jobname + '.sbatch'), 'w') as f:
        #    f.write(sbatch_template.format(jobname=jobname))
        with open(path.join(directory, jobname + '.bsub'), 'w') as f:
            f.write(bsub_template.format(jobname=jobname))
            #f.write(bsub_restart_template.format(jobname=jobname))

Universe Machine



In [11]:

    
tmp_cfg = copy.deepcopy(orig_cfg)
directory = path.join(cov_path, "mcmc/config/")
output_dir = path.join(emu_path, "PearceMCMC/")
jobname_template = "UniverseMachine_wp_ds_rmin_{rmin}_{emu_name}"

for rmin in [None, 0.5, 1.0, 2.0, 5.0]:
    for emu_fname, emu_name, emu_cov in zip(emu_fnames, emu_names, emu_cov_fnames):
        
        if rmin is not None:
            tmp_cfg['emu']['fixed_params'] = {'z': 0.0, 'rmin':rmin}
            
        tmp_cfg['emu']['training_file'] = emu_fname
        tmp_cfg['emu']['emu_type'] = ['LemonPepperWet' for i in xrange(len(emu_fname))]
        tmp_cfg['emu']['emu_cov_fname'] = emu_cov
        
        tmp_cfg['data']['true_data_fname']= [path.join(cov_path, 'mock_measurements/UMmock_wp.npy'),
                                             path.join(cov_path, 'mock_measurements/UMmock_ds.npy')]
        tmp_cfg['data']['true_cov_fname'] = meas_cov_fname
        
        tmp_cfg['data']['obs']['obs'] = ['wp','ds']
        tmp_cfg['data']['obs']['rbins'] = list(r_bins)
        

        tmp_cfg['chain']['nsteps'] = n_steps
        tmp_cfg['chain']['nwalkers'] = n_walkers
        tmp_cfg['chain']['mcmc_type'] = 'normal'

        
        tmp_cfg['data']['sim']['sim_hps']['system'] = 'ki-ls'
        tmp_cfg['data']['cov']['emu_cov_fname'] = tmp_cfg['emu']['emu_cov_fname'] 
        tmp_cfg['data']['cov']['meas_cov_fname'] = tmp_cfg['data']['true_cov_fname']
        
        jobname = jobname_template.format(rmin=rmin, emu_name=emu_name)
        tmp_cfg['fname'] = path.join(output_dir, jobname+'.hdf5')

        with open(path.join(directory, jobname +'.yaml'), 'w') as f:
            yaml.dump(tmp_cfg, f)

        #with open(path.join(directory, jobname + '.sbatch'), 'w') as f:
        #    f.write(sbatch_template.format(jobname=jobname))
        with open(path.join(directory, jobname + '.bsub'), 'w') as f:
            #f.write(bsub_template.format(jobname=jobname))
            f.write(bsub_restart_template.format(jobname=jobname))

HOD



In [18]:

    
#orig_cfg_fname = '/u/ki/swmclau2//Git/pearce/bin/mcmc/nh_gg_sham_hsab_mcmc_config.yaml'
with open(orig_cfg_fname, 'r') as yamlfile:
    orig_cfg = yaml.load(yamlfile)

orig_cfg



In [20]:

    
tmp_cfg = copy.deepcopy(orig_cfg)

directory = path.join(cov_path, "mcmc/config/")
output_dir = path.join(emu_path, "PearceMCMC/")

jobname_template = "HOD_wp_ds_rmin_{rmin}_{emu_name}"#_fixed_HOD"

for rmin in [None, 0.5, 1.0, 2.0]:
    for emu_fname, emu_name, emu_cov in zip(emu_fnames, emu_names, emu_cov_fnames):
        
        if rmin is not None:
            tmp_cfg['emu']['fixed_params'] = {'z': 0.0, 'rmin':rmin}
            
        tmp_cfg['emu']['training_file'] = emu_fname
        tmp_cfg['emu']['emu_type'] = ['LemonPepperWet' for i in xrange(len(emu_fname))]
        tmp_cfg['emu']['emu_cov_fname'] = emu_cov
        
        tmp_cfg['data']['obs']['obs'] = ['wp','ds']
        tmp_cfg['data']['obs']['rbins'] = list(r_bins)
        
        tmp_cfg['data']['true_data_fname']= [path.join(cov_path, 'mock_measurements/HODmock_wp.npy'),
                                             path.join(cov_path, 'mock_measurements/HODmock_ds.npy')]
        tmp_cfg['data']['true_cov_fname'] = meas_cov_fname
        
        
        tmp_cfg['data']['cov']['meas_cov_fname'] = tmp_cfg['data']['true_cov_fname']
        tmp_cfg['data']['cov']['emu_cov_fname'] = tmp_cfg['emu']['emu_cov_fname'] # TODO make this not be redundant
        
        jobname = jobname_template.format(rmin=rmin, emu_name=emu_name)
        tmp_cfg['fname'] = path.join(output_dir, jobname+'.hdf5')
        
        tmp_cfg['sim']= {'gal_type': 'HOD',
       'hod_name': 'zheng07',
       'hod_params': {'alpha': 1.083,
        'logM0': 13.2,
        'logM1': 14.2,
        'sigma_logM': 0.2,
        'conc_gal_bias': 1.0},
       'nd': '5e-4',
       'scale_factor': 1.0,
       'min_ptcl': 100, 
       'sim_hps': {'boxno': 1,
        'downsample_factor': 1e-2,
        'particles': True,
        'realization': 0,
        'system': 'ki-ls'},
       'simname': 'testbox'}
        
        # TODO i shouldnt have to specify this this way
        tmp_cfg['data']['sim'] = tmp_cfg['sim']
        
        tmp_cfg['chain']['nwalkers'] = n_walkers
        tmp_cfg['chain']['nsteps'] = n_steps
        tmp_cfg['chain']['mcmc_type'] = 'normal'
        
        # fix params during MCMC    
        #tmp_cfg['chain']['fixed_params'].update(tmp_cfg['sim']['hod_params'])
        
        #try:
        #    del tmp_cfg['data']['true_data_fname']
        #    del tmp_cfg['data']['true_cov_fname']
        #except KeyError:
        #    pass

        with open(path.join(directory, jobname +'.yaml'), 'w') as f:
            yaml.dump(tmp_cfg, f)

        #with open(path.join(directory, jobname + '.sbatch'), 'w') as f:
        #    f.write(sbatch_template.format(jobname=jobname))
            
        with open(path.join(directory, jobname + '.bsub'), 'w') as f:
            f.write(bsub_template.format(jobname=jobname))
            #f.write(bsub_restart_template.format(jobname=jobname))

HSAB HOD



In [ ]:

    
cosmo_pnames = ['ombh2', 'omch2', 'w0', 'ns', 'H0', 'Neff', 'sigma8']
cosmo_true_vals = [0.022762900000000003,
 0.11283,
 -0.861513,
 0.971515,
 65.7317,
 3.2,
 0.7542319000000001] # test box 1

tmp_cfg = copy.deepcopy(orig_cfg) directory = path.join(cov_path, "mcmc/config/") output_dir = path.join(emu_path, "PearceMCMC/") jobname_template = "CAB_HOD_fixed_cosmo_nfw_conc_wp_ds_rmin_{rmin}_{emu_name}"#_fixed_HOD" for rmin in [None, 0.5, 1.0, 2.0]: for emu_fname, emu_name, emu_cov in zip(emu_fnames, emu_names, emu_cov_fnames): if rmin is not None: tmp_cfg['emu']['fixed_params'] = {'z': 0.0, 'rmin':rmin} tmp_cfg['emu']['training_file'] = emu_fname tmp_cfg['emu']['emu_type'] = ['LemonPepperWet' for i in xrange(len(emu_fname))] tmp_cfg['emu']['emu_cov_fname'] = emu_cov tmp_cfg['data']['obs']['obs'] = ['wp','ds'] tmp_cfg['data']['obs']['rbins'] = list(r_bins) #tmp_cfg['data']['true_data_fname']= [path.join(cov_path, 'mock_measurements/HODmock_wp.npy'), # path.join(cov_path, 'mock_measurements/HODmock_ds.npy')] tmp_cfg['data']['true_cov_fname'] = meas_cov_fname tmp_cfg['data']['cov']['meas_cov_fname'] = tmp_cfg['data']['true_cov_fname'] tmp_cfg['data']['cov']['emu_cov_fname'] = tmp_cfg['emu']['emu_cov_fname'] # TODO make this not be redundant jobname = jobname_template.format(rmin=rmin, emu_name=emu_name) tmp_cfg['fname'] = path.join(output_dir, jobname+'.hdf5') tmp_cfg['chain']['fixed_params'] = 'cosmo' tmp_cfg['sim']= {'gal_type': 'HOD', 'hod_name': 'hsabZheng07', 'hod_params': {'alpha': 1.083, 'logM0': 13.2, 'logM1': 14.2, 'sigma_logM': 0.2, 'conc_gal_bias': 1.0, 'mean_occupation_centrals_assembias_param1': 1.0, 'mean_occupation_satellites_assembias_param1': -1.0}, 'cosmo_params': dict(zip(cosmo_pnames, cosmo_true_vals)), 'nd': '5e-4', 'scale_factor': 1.0, 'min_ptcl': 100, 'sim_hps': {'boxno': 1, 'downsample_factor': 1e-2, 'particles': True, 'realization': 1, 'system': 'ki-ls', 'sec_haloprop_key': 'halo_nfw_conc'}, 'simname': 'testbox'} # TODO i shouldnt have to specify this this way tmp_cfg['data']['sim'] = tmp_cfg['sim'] tmp_cfg['chain']['nwalkers'] = n_walkers tmp_cfg['chain']['nsteps'] = n_steps tmp_cfg['chain']['mcmc_type'] = 'normal' # fix params during MCMC #tmp_cfg['chain']['fixed_params'].update(tmp_cfg['sim']['hod_params']) try: del tmp_cfg['data']['true_data_fname'] del tmp_cfg['data']['true_cov_fname'] except KeyError: pass with open(path.join(directory, jobname +'.yaml'), 'w') as f: yaml.dump(tmp_cfg, f) #with open(path.join(directory, jobname + '.sbatch'), 'w') as f: # f.write(sbatch_template.format(jobname=jobname)) with open(path.join(directory, jobname + '.bsub'), 'w') as f: f.write(bsub_template.format(jobname=jobname)) #f.write(bsub_restart_template.format(jobname=jobname))

tmp_cfg = copy.deepcopy(orig_cfg) directory = path.join(cov_path, "mcmc/config/") output_dir = path.join(emu_path, "PearceMCMC/") jobname_template = "CAB_HOD_fixed_cosmo_lsd_wp_ds_rmin_{rmin}_{emu_name}"#_fixed_HOD" for rmin in [None, 0.5, 1.0, 2.0]: for emu_fname, emu_name, emu_cov in zip(emu_fnames, emu_names, emu_cov_fnames): if rmin is not None: tmp_cfg['emu']['fixed_params'] = {'z': 0.0, 'rmin':rmin} tmp_cfg['emu']['training_file'] = emu_fname tmp_cfg['emu']['emu_type'] = ['LemonPepperWet' for i in xrange(len(emu_fname))] tmp_cfg['emu']['emu_cov_fname'] = emu_cov tmp_cfg['data']['obs']['obs'] = ['wp','ds'] tmp_cfg['data']['obs']['rbins'] = list(r_bins) #tmp_cfg['data']['true_data_fname']= [path.join(cov_path, 'mock_measurements/HODmock_wp.npy'), # path.join(cov_path, 'mock_measurements/HODmock_ds.npy')] tmp_cfg['data']['true_cov_fname'] = meas_cov_fname tmp_cfg['data']['cov']['meas_cov_fname'] = tmp_cfg['data']['true_cov_fname'] tmp_cfg['data']['cov']['emu_cov_fname'] = tmp_cfg['emu']['emu_cov_fname'] # TODO make this not be redundant jobname = jobname_template.format(rmin=rmin, emu_name=emu_name) tmp_cfg['fname'] = path.join(output_dir, jobname+'.hdf5') tmp_cfg['chain']['fixed_params'] = 'cosmo' tmp_cfg['sim']= {'gal_type': 'SHAM', 'nd': '5e-4', 'scale_factor': 1.0, 'min_ptcl': 100, 'simname': 'mdpl2'} tmp_cfg['sim']= {'gal_type': 'HOD', 'hod_name': 'hsabZheng07', 'hod_params': {'alpha': 1.083, 'logM0': 13.2, 'logM1': 14.2, 'sigma_logM': 0.2, 'conc_gal_bias': 1.0, 'mean_occupation_centrals_assembias_param1': 1.0, 'mean_occupation_satellites_assembias_param1': -1.0}, 'cosmo_params': dict(zip(cosmo_pnames, cosmo_true_vals)), 'nd': '5e-4', 'scale_factor': 1.0, 'min_ptcl': 100, 'sim_hps': {'boxno': 1, 'downsample_factor': 1e-2, 'particles': True, 'realization': 1, 'system': 'ki-ls', 'sec_haloprop_key': 'halo_local_density_10'}, 'simname': 'testbox'} # TODO i shouldnt have to specify this this way tmp_cfg['data']['sim'] = tmp_cfg['sim'] tmp_cfg['chain']['nwalkers'] = n_walkers tmp_cfg['chain']['nsteps'] = n_steps tmp_cfg['chain']['mcmc_type'] = 'normal' # fix params during MCMC #tmp_cfg['chain']['fixed_params'].update(tmp_cfg['sim']['hod_params']) try: del tmp_cfg['data']['true_data_fname'] del tmp_cfg['data']['true_cov_fname'] except KeyError: pass with open(path.join(directory, jobname +'.yaml'), 'w') as f: yaml.dump(tmp_cfg, f) #with open(path.join(directory, jobname + '.sbatch'), 'w') as f: # f.write(sbatch_template.format(jobname=jobname)) with open(path.join(directory, jobname + '.bsub'), 'w') as f: f.write(bsub_template.format(jobname=jobname)) #f.write(bsub_restart_template.format(jobname=jobname))



In [ ]: