checking files from bogus branch

Author: jtwhite79

benchmarks/basic_tests.py

@@ -2125,10 +2125,13 @@ def parse_pst_test():
 
 
 
+
+
 if __name__ == "__main__":
     #parse_pst_test()
     #basic_test()
-    nonascii_path_test()
+    #mf6_v5_glm_test()
+    #nonascii_path_test()
 
     #mf6_v5_ies_nonpersistent_test()
     #large_fake_test()
@@ -2143,7 +2146,7 @@ def parse_pst_test():
     #plot_collapse_invest()
 
     #run()
-    #mf6_v5_ies_test()
+    
     #prep_ends()
     #sweep_bin_test()
     # mf6_v5_sen_test()
@@ -2185,7 +2188,7 @@ def parse_pst_test():
     #shutil.copy2(os.path.join("..","exe","windows","x64","Debug","pestpp-ies.exe"),os.path.join("..","bin","win","pestpp-ies.exe"))
     #tplins1_test()
 
-    #fr_timeout_test()
+    fr_timeout_test()
     #mf6_v5_ies_test()
     #mf6_v5_sen_test()
 

benchmarks/glm_test_base.py

@@ -1152,18 +1152,47 @@ def multimodal_test():
     assert phidf.iteration.max() == pst.control_data.noptmax
     assert phidf.loc[phidf.index[-1],"min"] < 0.1
 
-    pst.pestpp_options["ies_multimodal_alpha"] = 1.0
+    pst.pestpp_options["ies_multimodal_alpha"] = 0.0
     pst.write(os.path.join(test_d, "mm1.pst"))
     m_d = os.path.join(model_d, "master_base_{0}".format(func))
     #pyemu.os_utils.start_workers(test_d, exe_path, "mm1.pst", worker_root=model_d, num_workers=35, master_dir=m_d)
     if os.path.exists(m_d):
         shutil.rmtree(m_d)
     shutil.copytree(test_d,m_d)
     pyemu.os_utils.run("{0} mm1.pst".format(exe_path),cwd=m_d)
-    
+
+
     phidf = pd.read_csv(os.path.join(m_d,"mm1.phi.actual.csv"))
     assert phidf.iteration.max() == pst.control_data.noptmax
 
+
+    pst.pestpp_options["ies_multimodal_alpha"] = 1.0
+    pst.write(os.path.join(test_d, "mm1.pst"))
+    m_d = os.path.join(model_d, "master_allreals_{0}".format(func))
+    #pyemu.os_utils.start_workers(test_d, exe_path, "mm1.pst", worker_root=model_d, num_workers=35, master_dir=m_d)
+    if os.path.exists(m_d):
+        shutil.rmtree(m_d)
+    shutil.copytree(test_d,m_d)
+    pyemu.os_utils.run("{0} mm1.pst".format(exe_path),cwd=m_d)
+
+    phidf1 = pd.read_csv(os.path.join(m_d,"mm1.phi.actual.csv"))
+    assert phidf1.iteration.max() == pst.control_data.noptmax
+    
+    pst.pestpp_options["ies_multimodal_alpha"] = 0.99
+    pst.write(os.path.join(test_d, "mm1.pst"))
+    m_d = os.path.join(model_d, "master_almostallreals_{0}".format(func))
+    #pyemu.os_utils.start_workers(test_d, exe_path, "mm1.pst", worker_root=model_d, num_workers=35, master_dir=m_d)
+    if os.path.exists(m_d):
+        shutil.rmtree(m_d)
+    shutil.copytree(test_d,m_d)
+    pyemu.os_utils.run("{0} mm1.pst".format(exe_path),cwd=m_d)
+    phidf2 = pd.read_csv(os.path.join(m_d,"mm1.phi.actual.csv"))
+    assert phidf2.iteration.max() == pst.control_data.noptmax
+
+    diff = np.abs(phidf1["mean"].iloc[-1] - phidf2["mean"].iloc[-1])
+    print(diff)
+    assert diff < 1e-6
+
     pst.pestpp_options["ies_multimodal_alpha"] = 0.1
     pst.pestpp_options["ies_num_threads"] = 4
     pst.pestpp_options["ies_include_base"] = True
@@ -4948,14 +4977,47 @@ def tenpar_fixed_transform_test():
     assert diff.sum() < 1e-6
 
 
+def large_invest():
+    t_d = os.path.join("temp","template")
+    if os.path.exists(t_d):
+        shutil.rmtree(t_d)
+    os.makedirs(t_d)
+    npar = 30000
+    nobs = 60000
+    pnames = ["ppppppppppppppppppppppppppppppppppp{0:07d}".format(i) for i in range(npar)]
+
+    onames = ["ooooooooooooooooooooooooooooooooooo{0:07d}".format(i) for i in range(nobs)]
+    pst = pyemu.Pst.from_par_obs_names(pnames,onames)
+    obs = pst.observation_data
+    obs["obsval"] = np.random.normal(0,1,nobs)
+    par = pst.parameter_data
+    par["parval1"] = 1.0
+    par["parubnd"] = 1.1
+    par["parlbnd"] = 0.9
+    
+    pst.pestpp_options["check_tplins"] = False
+    pst.pestpp_options["ies_num_reals"] = 600
+    pst.pestpp_options["ies_multimodal_alpha"] = 0.75
+    pst.pestpp_options["save_dense"] = True
+    pst.pestpp_options["ies_num_threads"] = 10
+    pst.model_command = "ls"
+    pst.control_data.noptmax = 1
+    pst.write(os.path.join(t_d,"pest.pst"),version=2)
+
+    pyemu.os_utils.run("{0} pest.pst /e".format(exe_path),cwd=t_d)
+
 
-if __name__ == "__main__":
-    tenpar_fixed_transform_test()
 
+
+
+if __name__ == "__main__":
+    #large_invest()
+    #tenpar_fixed_transform_test()
+    #tenpar_reg_factor_test()
     #tenpar_ext_run_mgr_test()
     #freyberg_pdc_test()
     #tenpar_mean_iter_test()
-    #tenpar_reinflate_num_reals_2_test()
+    #tenpar_reinflate_num_reals_test()
     #freyberg_reinflate_num_reals_invest()
     #compared_freyberg_inflate_runs()
     #tenpar_mean_iter_test_sched()
@@ -4969,7 +5031,7 @@ def tenpar_fixed_transform_test():
     #tenpar_reg_factor_test()
     #tenpar_high_phi_test()
     #tenpar_iqr_bad_phi_sigma_test()
-    #multimodal_test()
+    multimodal_test()
     #plot_mm1_sweep_results()
     #plot_mm1_results()
     #plot_mm1_results_seq()

benchmarks/ies_test_part4.py

@@ -1,18 +1,15 @@
 pcf version=2
 * control data keyword
 pestmode                                 estimation
-noptmax                                 4
+noptmax                                 2
 svdmode                                 1
 maxsing                          10000000
 eigthresh                           1e-06
 eigwrite                                1
 ies_par_en                     prior.jcb
 forecasts                      headwater_20171130,tailwater_20161130,trgw_0_9_1_20161130
-n_iter_super                   4
-n_iter_base                    -1
 glm_num_reals                  200
 glm_normal_form                prior
-max_n_super                    36
 * parameter groups external
 freyberg6_run_glm.pargrp_data.csv
 * parameter data external

benchmarks/mf6_freyberg/template/freyberg6_run_glm.pst

@@ -442,6 +442,13 @@ def stack_test():
     pst = pyemu.Pst(pst_file)
     par = pst.parameter_data
     par.loc[par.partrans=="fixed","partrans"] = "log"
+    obs = pst.observation_data
+    obs.loc["q1","obsval"] = 100
+    obs.loc["q1","obgnme"] = "equal_to"
+    obs.loc["q1","weight"] = 1.0
+    
+    pst.pestpp_options.pop("opt_constraint_groups",None)
+    pst.pestpp_options.pop("base_jacobian",None)
     pst.pestpp_options["opt_risk"] = 0.1
     pst.pestpp_options["opt_stack_size"] = 10
     pst.control_data.noptmax = 1
@@ -489,7 +496,7 @@ def stack_test():
     if os.path.exists(d):
         shutil.rmtree(d)
     shutil.copytree(os.path.join("opt_dewater_chance", "template"), d)
-    pst.pestpp_options.pop("base_jacobian")
+    pst.pestpp_options.pop("base_jacobian",None)
     pst.write(os.path.join(d,"test.pst"))
     pyemu.os_utils.run("{0} {1}".format(exe_path, "test.pst"), cwd=d)   
     rec4 = os.path.join(d,"test.rec")
@@ -623,8 +630,6 @@ def fosm_invest():
         print(jcb_file,sc.get_forecast_summary().loc[:,"prior_var"].apply(lambda x: np.sqrt(x)))
 
 
-
-
 if __name__ == "__main__":
     #fosm_invest()
     #startworker()

benchmarks/opt_test.py

@@ -64,10 +64,10 @@ def basic_sqp_test():
 
     assert os.path.exists(os.path.join(m_d,"freyberg6_run_sqp.0.par.csv"))
     df = pd.read_csv(os.path.join(m_d,"freyberg6_run_sqp.0.par.csv"),index_col=0)
-    assert df.shape == (pst.pestpp_options["sqp_num_reals"],pst.npar),str(df.shape)
+    assert df.shape == (pst.pestpp_options["sqp_num_reals"] + 1,pst.npar),str(df.shape)
     assert os.path.exists(os.path.join(m_d,"freyberg6_run_sqp.0.obs.csv"))
     df = pd.read_csv(os.path.join(m_d,"freyberg6_run_sqp.0.obs.csv"),index_col=0)
-    assert df.shape == (pst.pestpp_options["sqp_num_reals"],pst.nobs),str(df.shape)
+    assert df.shape == (pst.pestpp_options["sqp_num_reals"] + 1,pst.nobs + pst.nprior),str(df.shape)
 
 
 def rosenbrock_setup(version,initial_decvars=1.6,constraints=False,constraint_exp="one_linear"):
@@ -328,54 +328,64 @@ def dewater_basic_test():
     pst.write(os.path.join(t_d,case+".pst"))
     #pyemu.os_utils.run("{0} {1}.pst".format(exe_path,case),cwd=t_d)
     m_d = os.path.join(model_d, "master2")
-    pyemu.os_utils.start_workers(t_d, exe_path, case + ".pst", num_workers=10, worker_root=model_d,
-                                 master_dir=m_d)
-    assert os.path.exists(os.path.join(m_d, case + ".base.par"))
-    assert os.path.exists(os.path.join(m_d, case + ".base.rei"))
-    assert os.path.exists(os.path.join(m_d, case + ".0.jcb"))
-    assert os.path.exists(os.path.join(m_d, case + ".1.jcb"))
-    assert os.path.exists(os.path.join(m_d, case + ".2.jcb"))
+    #pyemu.os_utils.start_workers(t_d, exe_path, case + ".pst", num_workers=10, worker_root=model_d,
+    #                             master_dir=m_d)
+    # assert os.path.exists(os.path.join(m_d, case + ".base.par"))
+    # assert os.path.exists(os.path.join(m_d, case + ".base.rei"))
+    # assert os.path.exists(os.path.join(m_d, case + ".0.jcb"))
+    # assert os.path.exists(os.path.join(m_d, case + ".1.jcb"))
+    # assert os.path.exists(os.path.join(m_d, case + ".2.jcb"))
 
     pst.pestpp_options["sqp_num_reals"] = 50
-    pst.control_data.noptmax = 3
+    pst.control_data.noptmax = 2
     pst.write(os.path.join(t_d, case + ".pst"))
     # pyemu.os_utils.run("{0} {1}.pst".format(exe_path,case),cwd=t_d)
     m_d = os.path.join(model_d, "master2_enopt")
     pyemu.os_utils.start_workers(t_d, exe_path, case + ".pst", num_workers=20, worker_root=model_d,
                                  master_dir=m_d)
 
+    for i in range(pst.control_data.noptmax+1):
+        assert os.path.exists(os.path.join(m_d, case + ".{0}.base.par".format(i)))
+        assert os.path.exists(os.path.join(m_d, case + ".{0}.base.rei".format(i)))
+        assert os.path.exists(os.path.join(m_d, case + ".{0}.obs.csv".format(i)))
+        assert os.path.exists(os.path.join(m_d, case + ".{0}.par.csv".format(i)))
+        
 
 
-def dewater_slp_opt_test():
-    model_d = "dewater"
-    local = True
-    if "linux" in platform.platform().lower() and "10par" in model_d:
-        # print("travis_prep")
-        # prep_for_travis(model_d)
-        local = False
+# def dewater_slp_opt_test():
+#     model_d = "dewater"
+#     local = True
+#     if "linux" in platform.platform().lower() and "10par" in model_d:
+#         # print("travis_prep")
+#         # prep_for_travis(model_d)
+#         local = False
 
-    t_d = os.path.join(model_d, "template")
+#     t_d = os.path.join(model_d, "template")
 
-    case = "dewater_pest.base"
-    pst = pyemu.Pst(os.path.join(t_d, case + ".pst"))
-    par = pst.parameter_data
-    dv_pars = par.loc[par.pargp == "q", "parnme"].tolist()[:3]
+#     case = "dewater_pest.base"
+#     pst = pyemu.Pst(os.path.join(t_d, case + ".pst"))
+#     par = pst.parameter_data
+#     dv_pars = par.loc[par.pargp == "q", "parnme"].tolist()[:3]
 
-    pst.add_pi_equation(par_names=dv_pars, pilbl="eq3", rhs=1000, obs_group="less_than")
-    pst.pestpp_options = {}
-    pst.pestpp_options["opt_dec_var_groups"] = "q"
-    pst.control_data.noptmax = 1
-    print(pst.prior_information)
-    pst.write(os.path.join(t_d, "test_opt.pst"))
-    pyemu.os_utils.run("{0} {1}.pst".format(exe_path.replace("-sqp","-opt"), "test_opt.pst"), cwd=t_d)
-
-
-    pst.parrep(os.path.join(t_d,"test_opt.par"))
-    pst.pestpp_options["hotstart_resfile"] = "test_opt.1.sim.rei"
-    pst.pestpp_options["base_jacobian"] = "test_opt.1.jcb"
-    pst.control_data.noptmax = 1
-    pst.write(os.path.join(t_d,"test_sqp.pst"))
-    pyemu.os_utils.run("{0} {1}.pst".format(exe_path, "test_sqp.pst"), cwd=t_d)
+#     pst.add_pi_equation(par_names=dv_pars, pilbl="eq3", rhs=1000, obs_group="less_than")
+#     pst.pestpp_options = {}
+#     pst.pestpp_options["opt_dec_var_groups"] = "q"
+#     pst.control_data.noptmax = 1
+#     print(pst.prior_information)
+#     pst.write(os.path.join(t_d, "test_opt.pst"))
+#     pyemu.os_utils.run("{0} {1}.pst".format(exe_path.replace("-sqp", "-opt"), "test_opt"), cwd=t_d)
+
+#     pst = pyemu.Pst(os.path.join(t_d, case + ".pst"))
+#     pst.add_pi_equation(par_names=dv_pars, pilbl="eq3", rhs=1000, obs_group="less_than")
+#     pst.pestpp_options = {}
+#     pst.pestpp_options["opt_dec_var_groups"] = "q"
+
+#     pst.parrep(os.path.join(t_d,"test_opt.par"))
+#     pst.pestpp_options["hotstart_resfile"] = "test_opt.1.sim.rei"
+#     pst.pestpp_options["base_jacobian"] = "test_opt.1.jcb"
+#     pst.control_data.noptmax = 1
+#     pst.write(os.path.join(t_d,"test_sqp.pst"))
+#     pyemu.os_utils.run("{0} {1}.pst".format(exe_path, "test_sqp"), cwd=t_d)
 
 
 def rosenc_test():
@@ -567,16 +577,79 @@ def plot_rosen(m_d):
                        "-i iter_%03d.png -vf scale=480X480  -y out.gif",cwd=m_d)
 
 
+def mf6_phiobs_invest():
+    model_d = "mf6_freyberg"
+
+    org_t_d = os.path.join("..","benchmarks",model_d,"template")
+    t_d = os.path.join(model_d,"template")
+    if os.path.exists(t_d):
+        shutil.rmtree(t_d)
+    shutil.copytree(org_t_d,t_d)
+    pst = pyemu.Pst(os.path.join(t_d,"freyberg6_run_ies.pst"))
+    pst.control_data.noptmax = 0
+    pst.write(os.path.join(t_d,"pest.pst"),version=2)
+    pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-sqp","-ies")),cwd=t_d)
+    pst = pyemu.helpers.add_phi_as_obs("pest.pst",pst_path=t_d)
+
+    pst.pestpp_options["opt_objective_function"] = "composite"
+    pst.observation_data.loc["composite","obgnme"] = "less_than"
+    lbdf = pst.add_pars_as_obs(pst_path=t_d,par_sigma_range=4,name_prefix="parlbnd-")
+    ubdf = pst.add_pars_as_obs(pst_path=t_d,par_sigma_range=4,name_prefix="parubnd-")
+    obs = pst.observation_data
+    print(obs.loc[lbdf.index,"greater_than"])
+    obs.loc[lbdf.index,"obgnme"] = "greater_than_parbound"
+    wpar = [w for w in lbdf.index if "wel" in w ]
+    obs.loc[wpar,"weight"] = 0
+
+    obs.loc[lbdf.index,"obsval"] = obs.loc[lbdf.index,"greater_than"]
+    obs.loc[ubdf.index,"obgnme"] = "less_than_parbound"
+    wpar = [w for w in ubdf.index if "wel" in w ]
+    obs.loc[wpar,"weight"] = 0
+    
+    obs.loc[ubdf.index,"obsval"] = obs.loc[ubdf.index,"less_than"]
+    par = pst.parameter_data
+    org_parval1 = par.parval1.copy()
+    par["parval1"] = par["parlbnd"]
+    pyemu.helpers.zero_order_tikhonov(pst)
+    par = pst.parameter_data
+    par["parval1"] = org_parval1
+    pi = pst.prior_information
+    pi["weight"] = 0
+    wpi = [w for w in pi.index if "wel" in w]
+    assert len(wpi) > 0
+    pi.loc[wpi,"weight"] = 1.0
+    pi.loc[wpi,"obgnme"] = 'greater_than'
+    #pi.loc[wpi,"obsval"] = np.log10(par.loc[wpi,"parlbnd"].values)
+
+    pst.dialate_par_bounds(1.5)
+    
+    pst.control_data.noptmax = 0
+    pst.write(os.path.join(t_d,"pest.pst"),version=2)
+    pyemu.os_utils.run("{0} pest.pst".format(exe_path),cwd=t_d)
+
+    pst.control_data.noptmax = 10
+    pst.pestpp_options["sqp_num_reals"] = 10
+    pst.write(os.path.join(t_d,"pest.pst"),version=2)
+    m_d = os.path.join(model_d,"master_sqp")
+    pyemu.os_utils.start_workers(t_d,exe_path,"pest.pst",num_workers=10,
+                                 master_dir=m_d,
+                                 worker_root=model_d)
+
+    
+
+
 if __name__ == "__main__":
 
     #if not os.path.exists(os.path.join("..","bin")):
     #    os.mkdir(os.path.join("..","bin"))
     #shutil.copy2(os.path.join("..","exe","windows","x64","Debug","pestpp-sqp.exe"),os.path.join("..","bin","pestpp-sqp.exe"))
-    basic_sqp_test()
+    #basic_sqp_test()
     #rosenbrock_single_linear_constraint(nit=1)
     #dewater_basic_test()
-    #dewater_slp_opt_test()
+    dewater_slp_opt_test()
     #rosenc_test()
     #m_d = rosenc_test()
     #m_d = os.path.join("mou_tests","master_rosenc_enopt")
     #plot_rosen(m_d)
+    #mf6_freyberg_test()
+    #mf6_phiobs_invest()