verify pyEMU null space projection with the freyberg problem


In [1]:
%matplotlib inline
import os
import shutil
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import pyemu

instaniate pyemu object and drop prior info. Then reorder the jacobian and save as binary. This is needed because the pest utilities require strict order between the control file and jacobian


In [2]:
mc = pyemu.MonteCarlo(jco="freyberg.jcb",verbose=False)
mc.drop_prior_information()
jco_ord = mc.jco.get(mc.pst.obs_names,mc.pst.par_names)
ord_base = "freyberg_ord"
jco_ord.to_binary(ord_base + ".jco")  
mc.pst.control_data.parsaverun = ' '
mc.pst.write(ord_base+".pst")

Draw some vectors from the prior and write the vectors to par files


In [3]:
# setup the dirs to hold all this stuff
par_dir = "prior_par_draws"
proj_dir = "proj_par_draws"
parfile_base = os.path.join(par_dir,"draw_")
projparfile_base = os.path.join(proj_dir,"draw_")
if os.path.exists(par_dir):
   shutil.rmtree(par_dir)
os.mkdir(par_dir)
if os.path.exists(proj_dir):
   shutil.rmtree(proj_dir)
os.mkdir(proj_dir)

# make some draws
mc.draw(10)

#write them to files
mc.parensemble.to_parfiles(parfile_base)

Run pnulpar


In [4]:
exe = os.path.join("exe","pnulpar.exe")
args = [ord_base+".pst","y","5","y","pnulpar_qhalfx.mat",parfile_base,projparfile_base]
in_file = os.path.join("misc","pnulpar.in")
with open(in_file,'w') as f:
    f.write('\n'.join(args)+'\n') 
os.system(exe + ' <'+in_file)


Out[4]:
0

In [5]:
pnul_en = pyemu.ParameterEnsemble(mc.pst)
parfiles =[os.path.join(proj_dir,f) for f in os.listdir(proj_dir) if f.endswith(".par")]
pnul_en.read_parfiles(parfiles)

In [7]:
pnul_en.loc[:,"fname"] = pnul_en.index
pnul_en.index = pnul_en.fname.apply(lambda x:str(int(x.split('.')[0].split('_')[-1])))
f = pnul_en.pop("fname")

In [8]:
pnul_en.sort(axis=1)
pnul_en


Out[8]:
parnme rch_1 rch_2 rcond00 rcond01 rcond02 rcond03 rcond04 rcond05 rcond06 rcond07 ... hkr39c05 hkr39c06 hkr39c07 hkr39c08 hkr39c09 hkr39c10 hkr39c11 hkr39c12 hkr39c13 hkr39c14
fname
1 1.021131 0.581702 3000.726 10565.550 1944.6290 10199.3200 2288.944 12160.750 3544.5870 11301.720 ... 31.432630 38.323910 0.991909 7.629075 11.505720 7.086335 5.025606 5.967037 10.233740 6.067576
10 0.948573 0.977763 21164.470 7157.494 578.3134 5742.3830 3535.766 26575.180 10011.7600 16393.100 ... 1.662408 11.412410 3.839063 4.597274 1.614462 20.955380 3.387948 13.040500 5.827925 5.944271
2 0.938769 0.749305 7315.771 27729.800 1888.3020 4416.2340 17986.100 4277.684 8835.6950 7273.200 ... 8.166737 3.303984 8.623650 2.298305 4.100882 16.094550 1.440205 10.245050 6.834493 1.726744
3 1.083537 1.261407 17110.940 6905.466 49043.7200 1274.0050 1705.403 49043.720 3260.6410 4505.166 ... 59.475120 11.253680 14.987500 15.959330 2.634054 14.594420 2.161142 4.793334 5.659204 1.542708
4 1.020887 1.032237 5731.537 10187.230 15761.1900 4240.0290 46682.580 5035.103 5020.7980 4531.105 ... 19.758450 2.243693 8.301189 13.365090 11.775890 4.385053 9.279671 1.053887 15.047550 5.423689
5 1.063933 1.464470 20876.430 11591.410 17529.8100 973.2425 5138.132 9965.900 8335.4100 15093.670 ... 6.364740 9.481560 5.183945 6.746825 2.999631 1.695221 8.222743 8.004017 1.547921 12.691170
6 0.988125 1.383985 49043.720 3587.045 13321.0300 7458.2450 7568.617 19033.520 493.6086 1246.822 ... 28.003940 9.303875 8.304618 48.251160 1.237535 2.928714 11.733070 16.439490 14.153460 2.707212
7 1.100000 0.968808 25286.370 35783.400 6981.4180 3968.8410 4953.955 4884.306 1221.3180 1140.399 ... 10.671780 33.638040 6.835930 4.783284 9.223482 42.977690 1.443917 59.475120 12.410590 59.475120
8 1.100000 0.698525 4157.452 9076.033 39803.3200 1566.1880 2367.845 44580.000 3570.5980 7676.086 ... 15.736610 3.170079 59.475120 2.344132 11.300920 1.470355 3.602373 1.232563 4.746572 8.048546
9 1.026231 0.942517 13428.960 14209.260 1240.0120 4396.6120 2178.109 5342.892 1571.7900 24505.650 ... 0.703678 18.957320 10.285270 9.331189 3.085356 0.661424 6.104861 14.483960 19.935100 30.723940

10 rows × 761 columns

Now for pyemu


In [9]:
print(mc.parensemble.islog)

en = mc.project_parensemble(nsing=5,inplace=False)
print(mc.parensemble.islog)


False
True

In [12]:
en.sort(axis=1)
en


Out[12]:
parnme rch_1 rch_2 rcond00 rcond01 rcond02 rcond03 rcond04 rcond05 rcond06 rcond07 ... hkr39c05 hkr39c06 hkr39c07 hkr39c08 hkr39c09 hkr39c10 hkr39c11 hkr39c12 hkr39c13 hkr39c14
1 1.021131 0.581702 3000.726463 10565.547233 1944.628929 10199.314027 2288.943865 12160.755433 3544.587426 11301.720721 ... 31.432627 38.323914 0.991909 7.629075 11.505720 7.086335 5.025606 5.967036 10.233731 6.067576
2 0.938769 0.749305 7315.771164 27729.791789 1888.302510 4416.234100 17986.096498 4277.684044 8835.694931 7273.199660 ... 8.166737 3.303984 8.623649 2.298305 4.100881 16.094547 1.440205 10.245054 6.834493 1.726744
3 1.083538 1.261407 17110.932958 6905.465937 49043.720000 1274.004881 1705.402969 49043.720000 3260.640885 4505.165878 ... 59.475120 11.253685 14.987499 15.959331 2.634054 14.594416 2.161142 4.793334 5.659204 1.542708
4 1.020887 1.032237 5731.536859 10187.236304 15761.182809 4240.028147 46682.586735 5035.102688 5020.797440 4531.104904 ... 19.758450 2.243694 8.301189 13.365088 11.775884 4.385053 9.279671 1.053887 15.047553 5.423689
5 1.063933 1.464470 20876.424961 11591.408534 17529.806742 973.242498 5138.131982 9965.903417 8335.409516 15093.673739 ... 6.364740 9.481560 5.183945 6.746825 2.999631 1.695220 8.222744 8.004017 1.547921 12.691172
6 0.988125 1.383985 49043.720000 3587.045166 13321.025922 7458.244852 7568.617071 19033.519435 493.608642 1246.821838 ... 28.003940 9.303875 8.304617 48.251169 1.237535 2.928714 11.733064 16.439489 14.153462 2.707212
7 1.100000 0.968808 25286.372123 35783.396774 6981.418075 3968.840904 4953.954661 4884.305650 1221.317694 1140.398896 ... 10.671780 33.638038 6.835930 4.783284 9.223482 42.977698 1.443917 59.475120 12.410586 59.475120
8 1.100000 0.698525 4157.452100 9076.033586 39803.327204 1566.188140 2367.844883 44580.000425 3570.597764 7676.086152 ... 15.736607 3.170079 59.475120 2.344131 11.300917 1.470356 3.602373 1.232563 4.746572 8.048546
9 1.026231 0.942517 13428.959116 14209.256159 1240.012202 4396.612103 2178.108145 5342.892283 1571.789708 24505.654382 ... 0.703678 18.957314 10.285276 9.331190 3.085357 0.661424 6.104862 14.483965 19.935103 30.723941
10 0.948573 0.977763 21164.465596 7157.494709 578.313404 5742.383085 3535.766094 26575.174781 10011.759142 16393.095058 ... 1.662408 11.412405 3.839063 4.597274 1.614461 20.955382 3.387948 13.040496 5.827925 5.944271

10 rows × 761 columns


In [13]:
pnul_en.sort(inplace=True)
en.sort(inplace=True)
diff = 100.0 * np.abs(pnul_en - en) / en
dmax = diff.max(axis=0)
dmax.sort(ascending=False,inplace=True)
dmax.plot()


Out[13]:
<matplotlib.axes._subplots.AxesSubplot at 0xff4e7f0>

In [ ]: