verify pyEMU null space projection with the freyberg problem


In [266]:
%matplotlib inline
import os
import shutil
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import pyemu

instaniate pyemu object and drop prior info. Then reorder the jacobian and save as binary. This is needed because the pest utilities require strict order between the control file and jacobian


In [267]:
mc = pyemu.MonteCarlo(jco="freyberg.jcb",verbose=False,forecasts=[])
mc.drop_prior_information()
jco_ord = mc.jco.get(mc.pst.obs_names,mc.pst.par_names)
ord_base = "freyberg_ord"
jco_ord.to_binary(ord_base + ".jco")  
mc.pst.control_data.parsaverun = ' '
mc.pst.write(ord_base+".pst")

Draw some vectors from the prior and write the vectors to par files


In [268]:
# setup the dirs to hold all this stuff
par_dir = "prior_par_draws"
proj_dir = "proj_par_draws"
parfile_base = os.path.join(par_dir,"draw_")
projparfile_base = os.path.join(proj_dir,"draw_")
if os.path.exists(par_dir):
   shutil.rmtree(par_dir)
os.mkdir(par_dir)
if os.path.exists(proj_dir):
   shutil.rmtree(proj_dir)
os.mkdir(proj_dir)
mc = pyemu.MonteCarlo(jco=ord_base+".jco")
# make some draws
mc.draw(10)
#for i in range(10):
#    mc.parensemble.iloc[i,:] = i+1
#write them to files
mc.parensemble.index = [str(i+1) for i in range(mc.parensemble.shape[0])]
mc.parensemble.to_parfiles(parfile_base)
mc.parensemble.shape


Out[268]:
(10, 761)

Run pnulpar


In [269]:
exe = os.path.join("pnulpar.exe")
args = [ord_base+".pst","y","1","y","pnulpar_qhalfx.mat",parfile_base,projparfile_base]
in_file = os.path.join("misc","pnulpar.in")
with open(in_file,'w') as f:
    f.write('\n'.join(args)+'\n') 
os.system(exe + ' <'+in_file)


Out[269]:
0

In [270]:
pnul_en = pyemu.ParameterEnsemble(mc.pst)
parfiles =[os.path.join(proj_dir,f) for f in os.listdir(proj_dir) if f.endswith(".par")]
pnul_en.read_parfiles(parfiles)

In [271]:
pnul_en.loc[:,"fname"] = pnul_en.index
pnul_en.index = pnul_en.fname.apply(lambda x:str(int(x.split('.')[0].split('_')[-1])))
f = pnul_en.pop("fname")

In [272]:
pnul_en.sort_index(axis=1,inplace=True)
pnul_en.sort_index(axis=0,inplace=True)
pnul_en


Out[272]:
parnme hkr00c00 hkr00c01 hkr00c02 hkr00c03 hkr00c04 hkr00c05 hkr00c06 hkr00c07 hkr00c08 hkr00c09 ... wf2_1 wf2_2 wf3_1 wf3_2 wf4_1 wf4_2 wf5_1 wf5_2 wf6_1 wf6_2
fname
1 10.302350 23.843160 6.620133 26.526310 0.594751 3.396889 0.594751 8.010879 33.428700 16.955270 ... 347.2585 433.011 303.2640 493.608 75.58456 57.6747 55.98720 116.6400 379.6522 552.830
10 0.594751 38.123330 5.596269 0.594751 3.180852 0.594751 0.594751 5.542466 32.177540 31.302280 ... 373.7491 433.709 356.6053 448.638 75.17855 115.7740 62.45971 101.6180 356.8950 459.103
2 0.594751 0.594751 7.361457 0.594751 0.594751 19.918090 12.830860 3.089787 0.594751 0.594751 ... 360.6368 544.598 351.3894 362.947 73.35143 67.6660 60.90968 66.8622 380.6407 505.464
3 0.594751 6.307704 13.567880 27.455170 20.149750 5.116127 16.624410 13.399200 26.852670 40.327250 ... 345.4191 327.880 339.0645 414.024 73.17263 67.9181 62.31440 90.6613 373.0379 279.559
4 24.686920 25.984680 10.441850 0.594751 0.594751 16.353810 21.965660 14.326890 0.594751 32.531980 ... 341.5155 284.696 358.5854 465.401 72.49356 44.8200 68.42880 70.3841 386.4073 467.936
5 0.594751 30.370610 0.594751 0.594751 20.304910 18.798440 15.142190 14.085080 1.899828 23.555190 ... 361.1883 480.874 367.7785 416.564 77.37764 60.1356 64.82001 107.0300 346.9478 445.312
6 13.864810 0.594751 0.594751 4.299358 30.367540 6.127359 6.727477 0.594751 0.594751 11.450430 ... 383.4623 291.575 332.5433 523.123 75.60550 54.8755 59.89003 70.0763 377.9833 507.121
7 0.594751 25.004600 36.617270 15.811540 0.594751 8.318873 11.769670 0.594751 12.764270 21.454840 ... 339.2773 639.821 331.8389 360.810 69.91741 69.5311 65.37884 70.9646 375.8613 437.085
8 0.594751 0.594751 10.094850 0.594751 26.317320 25.782280 24.942410 13.526740 10.436620 0.719666 ... 354.6512 605.961 346.0503 631.800 67.00251 80.7215 58.50130 53.7006 367.9061 586.415
9 25.847630 22.589160 9.099417 5.919446 0.594751 21.702860 0.594751 0.594751 0.594751 0.594751 ... 384.4837 450.653 367.0217 423.962 76.02603 46.6987 60.93617 82.4182 384.9697 416.943

10 rows × 761 columns

Now for pyemu


In [273]:
print(mc.parensemble.istransformed)
mc.parensemble._transform()
en = mc.project_parensemble(nsing=1,inplace=False)
print(mc.parensemble.istransformed)
#en._back_transform()


False
using 1 singular components
True

In [274]:
en.sort_index(axis=1,inplace=True)
en.sort_index(axis=0,inplace=True)
en


Out[274]:
parnme hkr00c00 hkr00c01 hkr00c02 hkr00c03 hkr00c04 hkr00c05 hkr00c06 hkr00c07 hkr00c08 hkr00c09 ... wf2_1 wf2_2 wf3_1 wf3_2 wf4_1 wf4_2 wf5_1 wf5_2 wf6_1 wf6_2
1 10.3023 23.8431 6.62013 26.5263 0.594751 3.39689 0.594751 8.01087 33.4287 16.9552 ... 347.258 433.011 303.264 493.608 75.5845 57.6747 55.9872 116.64 379.653 552.83
10 0.594751 38.1233 5.59627 0.594751 3.18085 0.594751 0.594751 5.54246 32.1776 31.3023 ... 373.749 433.709 356.605 448.638 75.1785 115.774 62.4597 101.618 356.895 459.103
2 0.594751 0.594751 7.36146 0.594751 0.594751 19.9181 12.8308 3.08979 0.594751 0.594751 ... 360.637 544.598 351.39 362.947 73.3515 67.666 60.9097 66.8622 380.64 505.464
3 0.594751 6.3077 13.5679 27.4552 20.1498 5.11612 16.6244 13.3992 26.8527 40.3273 ... 345.419 327.88 339.065 414.024 73.1726 67.9181 62.3144 90.6613 373.038 279.559
4 24.6869 25.9847 10.4418 0.594751 0.594751 16.3538 21.9656 14.3269 0.594751 32.5319 ... 341.515 284.696 358.585 465.401 72.4936 44.82 68.4288 70.3841 386.408 467.936
5 0.594751 30.3707 0.594751 0.594751 20.3049 18.7985 15.1422 14.0851 1.89983 23.5552 ... 361.189 480.874 367.779 416.564 77.3777 60.1356 64.82 107.03 346.947 445.312
6 13.8648 0.594751 0.594751 4.29936 30.3675 6.12736 6.72748 0.594751 0.594751 11.4504 ... 383.462 291.575 332.544 523.123 75.6055 54.8755 59.89 70.0763 377.984 507.121
7 0.594751 25.0046 36.6173 15.8116 0.594751 8.31887 11.7697 0.594751 12.7642 21.4549 ... 339.277 639.821 331.839 360.81 69.9174 69.5311 65.3788 70.9646 375.861 437.085
8 0.594751 0.594751 10.0949 0.594751 26.3173 25.7822 24.9424 13.5267 10.4367 0.719667 ... 354.651 605.961 346.051 631.8 67.0025 80.7215 58.5013 53.7006 367.906 586.415
9 25.8476 22.5891 9.09942 5.91944 0.594751 21.7028 0.594751 0.594751 0.594751 0.594751 ... 384.484 450.653 367.021 423.962 76.0261 46.6987 60.9362 82.4182 384.969 416.943

10 rows × 761 columns


In [275]:
#pnul_en.sort(inplace=True)
#en.sort(inplace=True)
diff = 100.0 * np.abs(pnul_en - en) / en

#diff[diff<1.0] = np.NaN
dmax = diff.max(axis=0)
dmax.sort_index(ascending=False,inplace=True)
dmax.plot(figsize=(10,10))
diff


Out[275]:
parnme hkr00c00 hkr00c01 hkr00c02 hkr00c03 hkr00c04 hkr00c05 hkr00c06 hkr00c07 hkr00c08 hkr00c09 ... wf2_1 wf2_2 wf3_1 wf3_2 wf4_1 wf4_2 wf5_1 wf5_2 wf6_1 wf6_2
fname
1 0.000371994 0.000119863 5.24489e-05 0.000166811 1.8667e-14 4.76828e-05 1.8667e-14 5.55717e-05 8.65994e-05 0.000267556 ... 7.82564e-05 8.94442e-05 0 4.06561e-06 4.77352e-05 4.98851e-05 0 0 0.000105558 8.7305e-05
10 1.8667e-14 8.87826e-06 4.87258e-05 1.8667e-14 2.87309e-05 1.8667e-14 1.8667e-14 2.27371e-05 0.000117178 2.24232e-05 ... 4.12354e-05 7.39452e-05 0.000118343 3.9247e-05 3.83881e-05 4.56204e-05 2.82118e-06 5.22897e-05 5.53193e-05 0.000100112
2 1.8667e-14 1.8667e-14 1.90846e-05 1.8667e-14 1.8667e-14 0.000102539 0.000264294 6.45414e-05 1.8667e-14 1.8667e-14 ... 6.26888e-05 2.27512e-05 0.000106632 9.19501e-05 6.50363e-05 4.07832e-05 2.31254e-05 4.34779e-05 6.21803e-05 9.15354e-05
3 1.8667e-14 4.59769e-05 0.000124033 1.97221e-05 0.000160881 4.88811e-05 0.000231828 0.000223657 6.08415e-05 9.99922e-05 ... 1.29576e-05 7.36101e-05 0.000116091 2.8109e-05 2.59538e-05 1.65994e-05 5.22694e-05 3.93569e-05 4.56511e-05 3.19681e-05
4 0.000171826 8.90566e-05 0.000317906 1.8667e-14 1.8667e-14 0.000131652 0.000117099 0.000170637 1.8667e-14 0.000132886 ... 0.000124507 0.000101075 1.0472e-05 4.69081e-05 1.21796e-05 0 0 5.08772e-05 7.29587e-05 2.99019e-05
5 1.8667e-14 0.000140147 1.8667e-14 1.8667e-14 0.000107603 0.000140482 0.000219295 0.000251206 3.727e-05 0.0001338 ... 6.13673e-05 6.61063e-05 7.14524e-05 2.58262e-05 4.08409e-05 3.33712e-05 3.98429e-05 1.43303e-05 0.000115338 9.79614e-05
6 0.000388358 1.8667e-14 1.8667e-14 2.41334e-05 0.00016587 2.02695e-05 2.87636e-05 1.8667e-14 1.8667e-14 0.000381637 ... 7.87411e-05 0.000103668 0.000144028 4.87406e-05 3.47834e-06 6.02201e-05 4.40627e-05 1.42932e-05 8.80591e-05 9.41318e-05
7 1.8667e-14 0.000160524 7.3312e-05 0.000258302 1.8667e-14 5.3653e-05 0.000329287 1.8667e-14 0.000301345 0.000146802 ... 4.4305e-05 1.15275e-05 0.000147879 8.16918e-05 1.64085e-06 5.79044e-05 5.00345e-05 4.07408e-05 3.50293e-05 9.24048e-05
8 1.8667e-14 1.8667e-14 0.0002782 1.8667e-14 9.48547e-05 0.00019385 0.000143958 0.000265487 0.000316273 7.42008e-05 ... 0.000101334 6.03786e-05 0.000120716 0 3.42514e-05 3.44417e-05 7.15133e-05 7.7375e-05 1.49778e-05 3.18205e-05
9 1.50607e-06 0.000152895 2.32502e-05 7.0322e-05 1.8667e-14 0.000104268 1.8667e-14 1.8667e-14 1.8667e-14 1.8667e-14 ... 1.74411e-05 8.37495e-05 8.29862e-05 0.00010978 5.98752e-05 7.04185e-05 3.67805e-05 2.59342e-05 9.57844e-05 2.94854e-05

10 rows × 761 columns


In [276]:
en.loc[:,"wf6_2"]


Out[276]:
1      552.83
10    459.103
2     505.464
3     279.559
4     467.936
5     445.312
6     507.121
7     437.085
8     586.415
9     416.943
Name: wf6_2, dtype: object

In [277]:
pnul_en.loc[:,"wf6_2"]


Out[277]:
fname
1     552.830
10    459.103
2     505.464
3     279.559
4     467.936
5     445.312
6     507.121
7     437.085
8     586.415
9     416.943
Name: wf6_2, dtype: float64

In [ ]: