Merge pull request #409 from jtwhite79/hotfix_iesfixed

Hotfix iesfixed

Author: jtwhite79

benchmarks/ies_test_part1.py

@@ -801,7 +801,7 @@ def tenpar_fixed_test2():
 def tenpar_fixed_test3():
     """tenpar fixed test 2"""
     model_d = "ies_10par_xsec"
-    test_d = os.path.join(model_d, "master_fixed2")
+    test_d = os.path.join(model_d, "master_fixed3")
     template_d = os.path.join(model_d, "template")
     pst = pyemu.Pst(os.path.join(template_d,"pest.pst"))
     pst.control_data.noptmax = 2
@@ -818,9 +818,9 @@ def tenpar_fixed_test3():
     pst.pestpp_options["ies_num_reals"] = 10
     pst.pestpp_options.pop("ies_par_en",None)
     pst.pestpp_options["ies_par_en"] = "par_fixed.csv"
-
+    scale = 10
     pst.parameter_data.loc["k_01","partrans"] = "fixed"
-    pst.parameter_data.loc["k_01","scale"] = 10.0
+    pst.parameter_data.loc["k_01","scale"] = scale
     pst.pestpp_options["ies_lambda_mults"] = 1.0
     pst.pestpp_options["lambda_scale_fac"] = 1.0
     pst.pestpp_options["ies_save_binary"] = False
@@ -854,15 +854,15 @@ def tenpar_fixed_test3():
         #df = df.iloc[:-1, :]
         #df.index = pe.index
         print(itr,pe.loc[:,"k_01"].values,oe.loc[:,"k_01"].values)
-        d = pe.loc[:,"k_01"].values - oe.loc[:,"k_01"].values
+        d = (pe.loc[:,"k_01"].values * scale) - oe.loc[:,"k_01"].values
         assert np.abs(d.max()) < 1e-6
         if prev is not None:
             d = prev - oe.loc[:,"k_01"].values
         assert np.abs(d.max()) < 1e-6
 
         prev = oe.loc[:,"k_01"].values
 
-    test_d = os.path.join(model_d, "master_fixed2")
+    test_d = os.path.join(model_d, "master_fixed3")
     template_d = os.path.join(model_d, "template")
     pst = pyemu.Pst(os.path.join(template_d,"pest.pst"))
     pst.control_data.noptmax = 2
@@ -881,7 +881,7 @@ def tenpar_fixed_test3():
     pst.pestpp_options["ies_par_en"] = "par_fixed.csv"
 
     pst.parameter_data.loc["k_01","partrans"] = "fixed"
-    pst.parameter_data.loc["k_01","scale"] = 10.0
+    pst.parameter_data.loc["k_01","scale"] = scale
     pst.pestpp_options["ies_lambda_mults"] = 1.0
     pst.pestpp_options["lambda_scale_fac"] = 1.0
     pst.pestpp_options["ies_save_binary"] = False
@@ -915,7 +915,7 @@ def tenpar_fixed_test3():
         #df = df.iloc[:-1, :]
         #df.index = pe.index
         print(itr,pe.loc[:,"k_01"].values,oe.loc[:,"k_01"].values)
-        d = pe.loc[:,"k_01"].values - oe.loc[:,"k_01"].values
+        d = (pe.loc[:,"k_01"].values*scale)- oe.loc[:,"k_01"].values
         assert np.abs(d.max()) < 1e-6
         if prev is not None:
             d = prev - oe.loc[:,"k_01"].values
@@ -2502,9 +2502,9 @@ def tenpar_ineq_test():
     #shutil.copy2(os.path.join("..", "exe", "windows", "x64", "Debug", "pestpp-ies.exe"),
     #             os.path.join("..", "bin", "win","pestpp-ies.exe"))
     #invest()
-    tenpar_ineq_test()
+    #tenpar_ineq_test()
     #tenpar_restart_similar_test()
-    #tenpar_fixed_test()
+    tenpar_fixed_test3()
     # tenpar_full_cov_test()
     # eval_freyberg_full_cov_reorder()
     #test_freyberg_full_cov_reorder()

benchmarks/ies_test_part4.py

@@ -3731,13 +3731,11 @@ def tenpar_fixed_restart_test():
     print(diff)
     assert np.abs(diff.values).max() < 1e-6
 
-
     #oe = pd.read_csv(os.path.join(test_d,"restart_obs.csv"),index_col=0)
     oe = pd.read_csv(os.path.join(test_d,"pest1.1.obs.csv"),index_col=0)
     oe.index = [str(i) for i in oe.index]
     r48_ovals = oe.loc["46",pst.obs_names]
 
-
     pst.control_data.noptmax = -2
     pst.pestpp_options.pop("ies_restart_obs_en",None)
     pst.pestpp_options.pop("ies_restart_par_en",None)
@@ -3779,6 +3777,7 @@ def tenpar_consistency_test():
     pst.pestpp_options["ies_no_noise"] = False
     pst.pestpp_options["save_binary"] = True
     pst.pestpp_options["ies_save_lambda_en"] = True
+    pst.pestpp_options["overdue_giveup_fac"] = 10000
 
     par = pst.parameter_data
     par.loc[pst.par_names[0],"partrans"] = "fixed"
@@ -4906,8 +4905,54 @@ def tenpar_ext_run_mgr_test():
     assert df.shape[0] == 25
 
 
+def tenpar_fixed_transform_test():
+    
+
+    model_d = "ies_10par_xsec"
+    test_d = os.path.join(model_d, "master_fixed_transform")
+    template_d = os.path.join(model_d, "test_template")
+
+    if not os.path.exists(template_d):
+        raise Exception("template_d {0} not found".format(template_d))
+
+    if os.path.exists(test_d):
+        shutil.rmtree(test_d)
+    shutil.copytree(template_d,test_d)
+    
+
+    pst_name = os.path.join(test_d, "pest.pst")
+    pst = pyemu.Pst(pst_name)
+    pst.control_data.noptmax = -1
+    pst.pestpp_options["ies_num_reals"] = 50 #hard coded later
+    par = pst.parameter_data
+    par["partrans"] = "none"
+    par.loc[pst.par_names[::2],"partrans"] = "fixed"
+    par.loc[pst.par_names[::2],"offset"] = 5
+
+    pst.write(os.path.join(test_d,"pest.pst"))
+
+    pyemu.os_utils.run("{0} pest.pst".format(exe_path),cwd=test_d)
+    pst = pyemu.Pst(pst_name)
+    pe0 = pst.ies.paren0.copy()
+    shutil.copy2(os.path.join(test_d,"pest.0.par.csv"),os.path.join(test_d,"restart_par.csv"))
+    pst.pestpp_options["ies_par_en"] = "restart_par.csv"
+    pst.write(os.path.join(test_d,"pest_trans.pst"))
+
+    pyemu.os_utils.run("{0} pest_trans.pst".format(exe_path),cwd=test_d)
+    pst = pyemu.Pst(pst_name)
+    pe1 = pst.ies.paren0.copy()
+    assert pe0.shape == pe1.shape
+    diff =  np.abs(pe0.values - pe1.values)
+    print(diff)
+    print(diff.sum())
+    assert diff.sum() < 1e-6
+    
+    
+
 if __name__ == "__main__":
-    tenpar_ext_run_mgr_test()
+    tenpar_fixed_transform_test()
+
+    #tenpar_ext_run_mgr_test()
     #freyberg_pdc_test()
     #tenpar_mean_iter_test()
     #tenpar_reinflate_num_reals_2_test()

benchmarks/mou_tests_2.py

@@ -2156,7 +2156,27 @@ def zdt1_fixed_scaleoffset_test():
 
     print(df.loc[others,:])
     assert np.all(df.loc[others,"val"].values <= 0)
+    dv0 = pd.read_csv(os.path.join(t_d,"zdt1.0.dv_pop.csv"),index_col=0)
+    assert dv0.shape == pe.shape
+    diff = np.abs(dv0.values - pe._df.loc[dv0.index,dv0.columns].values)
+    print(diff)
+    print(diff.sum())
+    assert diff.max() < 1e-10
+    pst.control_data.noptmax = 1
+    pst.write(os.path.join(t_d,"zdt1.pst"))
 
+    pyemu.os_utils.run("{0} {1}".format(exe_path,"zdt1.pst"),cwd=t_d)
+    dv1 = pd.read_csv(os.path.join(t_d,"zdt1.1.dv_pop.csv"),index_col=0)
+    for other in others:
+        #print(dv1.loc[:,other])
+        assert np.all(dv1.loc[:,other].values > 0)
+        #make sure that the values in the fixed columns are also in the initial pe
+        min_diff = 1e10
+        for val in dv1.loc[:,other].values:
+            min_diff = min(min_diff,np.abs((pe.loc[:,others] - val).values).min())
+            print(min_diff)
+            assert min_diff < 1e-10
+
 
 def zdt1_fixedtied_stack_test():
     t_d = mou_suite_helper.setup_problem("zdt1",True,True)
@@ -2695,7 +2715,7 @@ def mou_bound_handling_test():
 
 
 if __name__ == "__main__":
-    mou_bound_handling_test()
+    #mou_bound_handling_test()
     #basic_pso_test()
     #test_restart_all()
     #chance_consistency_test()
@@ -2709,7 +2729,7 @@ def mou_bound_handling_test():
     #plot_hosaki(m_d=os.path.join("mou_tests","hosaki_empcov_master_risk"))
     #basic_pso_test()
     #zdt1_fixedtied_stack_test()
-    #zdt1_fixed_scaleoffset_test()
+    zdt1_fixed_scaleoffset_test()
     #shutil.copy2(os.path.join("..","exe","windows","x64","Debug","pestpp-mou.exe"),os.path.join("..","bin","pestpp-mou.exe"))
     #invest()
     #plot()
@@ -2777,4 +2797,4 @@ def mou_bound_handling_test():
 
     #pop_sched_test()
     #simplex_invest_1()
-    pi_output_test()
+    #pi_output_test()

src/libs/pestpp_common/Ensemble.cpp

@@ -2229,7 +2229,7 @@ map<int,int> ParameterEnsemble::add_runs(RunManagerAbstract *run_mgr_ptr,const v
 			par_transform.active_ctl2model_ip(pars_real);
 		else if (tstat == ParameterEnsemble::transStatus::NUM)
 			par_transform.numeric2model_ip(pars_real);
-		replace_fixed(rname, pars_real);
+		replace_fixed(rname, pars_real, true);
 		nn = pars_real.get_notnormal_keys();
 		if (nn.size() > 0) {
             stringstream ss;
@@ -2433,19 +2433,7 @@ void ParameterEnsemble::prep_par_ensemble_after_read(map<string, int>& header_in
 			tied_names.push_back(name);
 		}
 	}
-//	vector<string> problems;
-//	for (auto& name : fixed_names)
-//    {
-//	    if ((pi.get_parameter_rec_ptr(name)->scale != 1.0) ||
-//                (pi.get_parameter_rec_ptr(name)->offset != 0.0))
-//        {
-//	        problems.push_back(name);
-//        }
-//    }
-//	if (problems.size())
-//    {
-//        throw_ensemble_error("the following fixed parameters have been passed values but have non-trivial scale/offset, which is not supported",problems);
-//    }
+
 	pfinfo.set_fixed_names(fixed_names);
 	fill_fixed(header_info, fixed_names);
 	save_fixed(fixed_names);
@@ -3392,8 +3380,7 @@ void ParameterEnsemble::to_csv_by_reals(ofstream &csv, bool write_header)
 		{
 			par_transform.active_ctl2numeric_ip(pars);
 		}
-		else if (tstat == transStatus::MODEL)
-		{
+		else if (tstat == transStatus::MODEL) {
 			par_transform.active_ctl2model_ip(pars);
 		}
 
@@ -3418,13 +3405,22 @@ void ParameterEnsemble::to_csv_by_reals(ofstream &csv, bool write_header)
 	}
 }
 
-void ParameterEnsemble::replace_fixed(string real_name,Parameters &pars)
+void ParameterEnsemble::replace_fixed(string real_name,Parameters &pars, bool to_model)
 {
 
 	map<string, double> rmap = pfinfo.get_real_fixed_values(real_name);
+	double val, offset, scale;
+
 	for (auto& r : rmap)
 	{
-		pars.update_rec(r.first, r.second);
+		val = r.second;
+		if (!to_model) {
+			scale = pest_scenario_ptr->get_ctl_parameter_info_ptr_4_mod()->get_parameter_rec_ptr(r.first)->scale;
+			offset = pest_scenario_ptr->get_ctl_parameter_info_ptr_4_mod()->get_parameter_rec_ptr(r.first)->offset;
+			val -= offset;
+			val /= scale;
+		}
+		pars.update_rec(r.first, val);
 	}
 
 
@@ -4102,6 +4098,8 @@ map<string, double> FixedParInfo::get_real_fixed_values(const string& rname)
 		return map<string, double>();
 	}
 	map<string, double> rmap;
+	double val;
+
 	for (auto& fi : fixed_info)
 	{
 		if (fi.second.find(rname) == fi.second.end())

src/libs/pestpp_common/Ensemble.h

@@ -208,7 +208,7 @@ class ParameterEnsemble : public Ensemble
 	transStatus tstat;
 	void save_fixed(vector<string>& fixed_names);
 	void fill_fixed(const map<string, int> &header_info, vector<string>& fixed_names);
-	void replace_fixed(string real_name,Parameters &pars);
+	void replace_fixed(string real_name,Parameters &pars,bool to_model=false);
 	void prep_par_ensemble_after_read(map<string,int>& header_info);
 	FixedParInfo pfinfo;
 };