Skip to content

Commit 950580a

Browse files
FDubathFDubath
committed
Merge branch 'release-2.1' into 'develop'
Release 2.1 See merge request PF-PHZ/PhosphorosCore!82
2 parents ae8c95c + bd1fc28 commit 950580a

File tree

12 files changed

+129
-37
lines changed

12 files changed

+129
-37
lines changed

‎PhzConfiguration/src/lib/CatalogTypeConfig.cpp‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,10 +45,14 @@ auto CatalogTypeConfig::getProgramOptions() -> std::map<std::string, OptionDescr
4545

4646
void CatalogTypeConfig::preInitialize(const UserValues& args) {
4747
regex expr{"[0-9a-zA-Z_]+"};
48-
auto type = args.at(CATALOG_TYPE).as<std::string>();
49-
if (!regex_match(type, expr)) {
50-
throw Elements::Exception() << "Malformed " << CATALOG_TYPE << " value (" << type
51-
<< "). Only letters numbers and the underscore are allowed.";
48+
if (args.count(CATALOG_TYPE) > 0) {
49+
auto type = args.at(CATALOG_TYPE).as<std::string>();
50+
if (!regex_match(type, expr)) {
51+
throw Elements::Exception() << "Malformed " << CATALOG_TYPE << " value (" << type
52+
<< "). Only letters numbers and the underscore are allowed.";
53+
}
54+
} else {
55+
throw Elements::Exception() << "Missing " << CATALOG_TYPE << " value.";
5256
}
5357
}
5458

‎PhzConfiguration/src/lib/ComputeSedWeightConfig.cpp‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,12 @@
2626
#include "ElementsKernel/Exception.h"
2727
#include "ElementsKernel/Logging.h"
2828
#include "PhzConfiguration/AuxDataDirConfig.h"
29+
#include <filesystem>
30+
31+
#include "PhzConfiguration/CatalogTypeConfig.h"
2932
#include "PhzConfiguration/FilterConfig.h"
3033
#include "PhzConfiguration/FilterProviderConfig.h"
34+
#include "PhzConfiguration/IntermediateDirConfig.h"
3135
#include "PhzConfiguration/PhotometryGridConfig.h"
3236
#include "PhzConfiguration/SedConfig.h"
3337
#include "PhzConfiguration/SedProviderConfig.h"
@@ -44,7 +48,8 @@ static const std::string SED_WEIGHT_OUTPUT{"SED-Weight-Output"};
4448
static const std::string SED_WEIGHT_SAMPLING{"SED-Weight-sampling"};
4549

4650
ComputeSedWeightConfig::ComputeSedWeightConfig(long manager_id) : Configuration(manager_id) {
47-
declareDependency<AuxDataDirConfig>();
51+
declareDependency<CatalogTypeConfig>();
52+
declareDependency<IntermediateDirConfig>();
4853
declareDependency<PhotometryGridConfig>();
4954
declareDependency<SedProviderConfig>();
5055
declareDependency<FilterConfig>();
@@ -57,9 +62,9 @@ auto ComputeSedWeightConfig::getProgramOptions() -> std::map<std::string, Option
5762

5863
{SED_WEIGHT_OUTPUT.c_str(), po::value<std::string>()->default_value("SedWeight.ascii"),
5964
"Path of the file into which output the SED weights. Relative path are relative to "
60-
"<AuxDataDir>/GenericPriors/SedWeight/"},
61-
{SED_WEIGHT_SAMPLING.c_str(), po::value<int>()->default_value(100000),
62-
"Number of sample for computing SED weight, if put to 0 all weight are set to 1"}
65+
"<intermediate_dir>/<catalog_type>/GenericPriors/SedWeight/"},
66+
{SED_WEIGHT_SAMPLING.c_str(), po::value<int>()->default_value(100000),
67+
"Number of sample for computing SED weight, if put to 0 all weight are set to 1"}
6368

6469
}}};
6570
}
@@ -72,9 +77,21 @@ void ComputeSedWeightConfig::initialize(const UserValues& args) {
7277
m_output_file = file_name;
7378
} else {
7479
// relative to
75-
boost::filesystem::path dir(getDependency<AuxDataDirConfig>().getAuxDataDir() / "GenericPriors" / "SedWeight");
76-
boost::filesystem::create_directory(dir);
77-
fs::path result = getDependency<AuxDataDirConfig>().getAuxDataDir() / "GenericPriors" / "SedWeight" / file_name;
80+
81+
boost::filesystem::path outer_dir(getDependency<IntermediateDirConfig>().getIntermediateDir() /
82+
getDependency<CatalogTypeConfig>().getCatalogType() / "GenericPriors");
83+
84+
if (!std::filesystem::exists(outer_dir.string())) {
85+
boost::filesystem::create_directory(outer_dir);
86+
}
87+
88+
boost::filesystem::path dir(getDependency<IntermediateDirConfig>().getIntermediateDir() /
89+
getDependency<CatalogTypeConfig>().getCatalogType() / "GenericPriors" / "SedWeight");
90+
if (!std::filesystem::exists(dir.string())) {
91+
boost::filesystem::create_directory(dir);
92+
}
93+
94+
fs::path result = dir / file_name;
7895
m_output_file = result.string();
7996
}
8097

@@ -95,6 +112,5 @@ int ComputeSedWeightConfig::getWeightSampling() const {
95112
return m_sampling;
96113
}
97114

98-
99115
} // namespace PhzConfiguration
100116
} // namespace Euclid

‎PhzConfiguration/src/lib/GenericGridPriorConfig.cpp‎

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,17 +22,22 @@
2222
* @author nikoapos
2323
*/
2424

25-
#include "PhzConfiguration/GenericGridPriorConfig.h"
25+
#include <filesystem>
26+
2627
#include "ElementsKernel/Logging.h"
2728
#include "GridContainer/serialize.h"
2829
#include "PhzConfiguration/AuxDataDirConfig.h"
30+
#include "PhzConfiguration/GenericGridPriorConfig.h"
2931
#include "PhzConfiguration/PriorConfig.h"
3032
#include "PhzDataModel/DoubleGrid.h"
3133
#include "PhzLikelihood/GenericGridPrior.h"
3234
#include "PhzLikelihood/SharedPriorAdapter.h"
3335
#include <CCfits/CCfits>
3436
#include <boost/filesystem.hpp>
3537

38+
#include "PhzConfiguration/CatalogTypeConfig.h"
39+
#include "PhzConfiguration/IntermediateDirConfig.h"
40+
3641
namespace po = boost::program_options;
3742
namespace fs = boost::filesystem;
3843

@@ -45,7 +50,8 @@ namespace PhzConfiguration {
4550

4651
GenericGridPriorConfig::GenericGridPriorConfig(long manager_id) : Configuration(manager_id) {
4752
declareDependency<PriorConfig>();
48-
declareDependency<AuxDataDirConfig>();
53+
declareDependency<CatalogTypeConfig>();
54+
declareDependency<IntermediateDirConfig>();
4955
}
5056

5157
auto GenericGridPriorConfig::getProgramOptions() -> std::map<std::string, OptionDescriptionList> {
@@ -78,8 +84,23 @@ void GenericGridPriorConfig::initialize(const UserValues& args) {
7884
for (auto& name : args.at(GENERIC_GRID_PRIOR).as<std::vector<std::string>>()) {
7985
fs::path filename{name};
8086
if (!filename.is_absolute()) {
81-
auto& aux_dir = getDependency<AuxDataDirConfig>().getAuxDataDir();
82-
filename = aux_dir / "GenericPriors" / filename;
87+
auto intermediate_dir = getDependency<IntermediateDirConfig>().getIntermediateDir();
88+
auto catalog_type = getDependency<CatalogTypeConfig>().getCatalogType();
89+
auto aux_dir = getDependency<AuxDataDirConfig>().getAuxDataDir();
90+
if (std::filesystem::exists(filename.string())) {
91+
// Local file
92+
logger.info("Find a file for the generic grid prior with path " + filename.string());
93+
} else if (std::filesystem::exists((intermediate_dir / catalog_type / "GenericPriors" / filename).string())) {
94+
// Intermediate result file
95+
filename = intermediate_dir / catalog_type / "GenericPriors" / filename;
96+
logger.info("Find a file for the generic grid prior with path " + filename.string());
97+
} else if (std::filesystem::exists((aux_dir / "GenericPriors" / filename).string())) {
98+
// Aux dir file (legacy)
99+
filename = aux_dir / "GenericPriors" / filename;
100+
logger.info("Find a file for the generic grid prior with path " + filename.string());
101+
} else {
102+
logger.error("No file was found for the generic grid prior");
103+
}
83104
}
84105
std::vector<PhzDataModel::DoubleGrid> grids = readGridsFromFile(filename);
85106
getDependency<PriorConfig>().addPrior(SharedPriorAdapter<GenericGridPrior>::factory(std::move(grids)));

‎PhzConfiguration/src/lib/IntermediateDirConfig.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ void IntermediateDirConfig::initialize(const UserValues& args) {
6868

6969
const fs::path& IntermediateDirConfig::getIntermediateDir() {
7070
if (getCurrentState() < Configuration::Configuration::State::INITIALIZED) {
71-
throw Elements::Exception() << "Call to getAuxDataDir() on a not initialized instance.";
71+
throw Elements::Exception() << "Call to getIntermediateDir() on a not initialized instance.";
7272
}
7373
return m_intermediate_dir;
7474
}

‎PhzConfiguration/src/lib/LuminosityPriorConfig.cpp‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -84,8 +84,8 @@ auto LuminosityPriorConfig::getProgramOptions() -> std::map<std::string, OptionD
8484
"If added, turn Luminosity Prior on (YES/NO, default: NO)"},
8585
{LUMINOSITY_PRIOR_EFFECTIVENESS.c_str(), po::value<double>()->default_value(1.),
8686
"A value in the range [0,1] showing how strongly to apply the prior"},
87-
{LUMINOSITY_PRIOR_PERMPC3.c_str(), po::value<std::string>()->default_value("YES"),
88-
"Set if the luminosity function is in 1/mpc³ (YES/NO, default: YES)"}}}};
87+
{LUMINOSITY_PRIOR_PERMPC3.c_str(), po::value<std::string>()->default_value(""),
88+
"Set if the luminosity function is in 1/Mpc³ (YES/NO)"}}}};
8989
}
9090

9191
void LuminosityPriorConfig::preInitialize(const UserValues& args) {
@@ -109,11 +109,8 @@ void LuminosityPriorConfig::initialize(const UserValues& args) {
109109
m_is_configured =
110110
args.count(LUMINOSITY_PRIOR) == 1 && args.find(LUMINOSITY_PRIOR)->second.as<std::string>().compare("YES") == 0;
111111

112-
m_permpc3 = args.count(LUMINOSITY_PRIOR_PERMPC3) == 1 &&
113-
args.find(LUMINOSITY_PRIOR_PERMPC3)->second.as<std::string>().compare("YES") == 0;
114-
115112
if (m_is_configured) {
116-
113+
logger.info() << "Luminosity Prior Configured";
117114
bool inMag = getDependency<LuminosityFunctionConfig>().isExpressedInMagnitude();
118115

119116
double scale_sampling_range_sigma = getDependency<ScaleFactorMarginalizationConfig>().getRangeInSigma();
@@ -140,13 +137,22 @@ void LuminosityPriorConfig::initialize(const UserValues& args) {
140137
PhzLikelihood::SharedPriorAdapter<PhzLuminosity::LuminosityPrior> prior{prior_ptr};
141138

142139
getDependency<PriorConfig>().addPrior(prior);
140+
141+
if (args.count(LUMINOSITY_PRIOR_PERMPC3) != 1 ||
142+
!(args.find(LUMINOSITY_PRIOR_PERMPC3)->second.as<std::string>().compare("YES") == 0 ||
143+
args.find(LUMINOSITY_PRIOR_PERMPC3)->second.as<std::string>().compare("NO") == 0)) {
144+
logger.error() << "Missing " << LUMINOSITY_PRIOR_PERMPC3 << " parameter.";
145+
throw Elements::Exception() << "Missing " << LUMINOSITY_PRIOR_PERMPC3 << " parameter.";
146+
} else {
147+
m_permpc3 = args.find(LUMINOSITY_PRIOR_PERMPC3)->second.as<std::string>().compare("YES") == 0;
148+
}
143149
}
144150

145151
// Volume prior manipulation
146152
if (m_is_configured && m_permpc3) {
147153
// force the volume prior on
148154
logger.info()
149-
<< "Luminosity Prior defined in 1/mpc³ need the Volume Prior on to be meaningful. Forcing the Volume Prior on.";
155+
<< "Luminosity Prior defined in 1/Mpc³ need the Volume Prior on to be meaningful. Forcing the Volume Prior on.";
150156
getDependency<VolumePriorConfig>().forceOn();
151157
}
152158
}

‎PhzConfiguration/src/lib/PhotometryGridConfig.cpp‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
#include <boost/archive/binary_iarchive.hpp>
2727
#include <boost/archive/text_iarchive.hpp>
2828
#include <fstream>
29+
#include <list>
2930

3031
#include "ElementsKernel/Exception.h"
3132
#include "ElementsKernel/Logging.h"
@@ -130,9 +131,11 @@ void PhotometryGridConfig::initialize(const UserValues& args) {
130131

131132
if (!same) {
132133
// Check that we have all the catalog photometries in the grid
134+
auto iter = m_info.filter_names.begin();
133135
for (auto& f : *filter_names) {
134-
if (std::count(m_info.filter_names.begin(), m_info.filter_names.end(), f) == 0) {
135-
throw Elements::Exception() << "Filter " << f << " missing from the model grid";
136+
iter = std::find(iter, m_info.filter_names.end(), f);
137+
if (iter == m_info.filter_names.end()) {
138+
throw Elements::Exception() << "Filter " << f << " missing from the model grid or not in the right order";
136139
}
137140
}
138141

‎PhzConfiguration/tests/src/GenericGridPriorConfig_test.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,8 @@ struct GenericGridPriorConfig_fixture : public ConfigManager_fixture {
7777

7878
options_map = registerConfigAndGetDefaultOptionsMap<GenericGridPriorConfig>();
7979
options_map[GENERIC_GRID_PRIOR].value() = boost::any{std::vector<std::string>{prior_file}};
80+
std::string toto{"toto"};
81+
options_map["catalog-type"].value() = boost::any{toto};
8082
}
8183
};
8284

‎PhzConfiguration/tests/src/PhotometryGridConfig_test.cpp‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,10 @@ struct PhotometryGridConfig_fixture : public ConfigManager_fixture {
4949
fs::path intermediate_dir = temp_dir.path() / "Intermediate";
5050
std::string catalog_type{"CatalogType"};
5151
std::string filename{"model_grid.dat"};
52-
fs::path relative = fs::path{"relative"} / filename;
53-
fs::path absolute = temp_dir.path() / "absolute" / filename;
54-
fs::path filter_mapping = temp_dir.path() / "filter-mapping.txt";
52+
fs::path relative = fs::path{"relative"} / filename;
53+
fs::path absolute = temp_dir.path() / "absolute" / filename;
54+
fs::path filter_mapping = temp_dir.path() / "filter-mapping.txt";
55+
fs::path filter_mapping_alt_order = temp_dir.path() / "filter-mapping-alt.txt";
5556

5657
std::map<std::string, po::variable_value> options_map{};
5758

@@ -93,6 +94,12 @@ struct PhotometryGridConfig_fixture : public ConfigManager_fixture {
9394
stream << "Filter2 FLUX_FILTER1 FLUXERR_FILTER1 0 3 NONE\n";
9495
stream << "Filter3 FLUX_FILTER3 FLUXERR_FILTER3 0 3 NONE\n";
9596
}
97+
{
98+
std::ofstream stream{filter_mapping_alt_order.string()};
99+
stream << "Filter1 FLUX_FILTER1 FLUXERR_FILTER1 0 3 NONE\n";
100+
stream << "Filter3 FLUX_FILTER3 FLUXERR_FILTER3 0 3 NONE\n";
101+
stream << "Filter2 FLUX_FILTER1 FLUXERR_FILTER1 0 3 NONE\n";
102+
}
96103

97104
options_map["intermediate-products-dir"].value() = boost::any(intermediate_dir.string());
98105
options_map["catalog-type"].value() = boost::any(catalog_type);
@@ -217,6 +224,18 @@ BOOST_FIXTURE_TEST_CASE(withFilterMapping_test, PhotometryGridConfig_fixture) {
217224
std::vector<std::string> expected{"Filter1", "Filter2", "Filter3"};
218225
BOOST_CHECK_EQUAL_COLLECTIONS(expected.begin(), expected.end(), used_filters.begin(), used_filters.end());
219226
}
227+
//-----------------------------------------------------------------------------
228+
229+
BOOST_FIXTURE_TEST_CASE(withFilterMapping_wrong_order_test, PhotometryGridConfig_fixture) {
230+
config_manager.registerConfiguration<PhotometryGridConfig>();
231+
config_manager.registerConfiguration<PhotometricBandMappingConfig>();
232+
config_manager.closeRegistration();
233+
234+
options_map["filter-mapping-file"].value() = boost::any(filter_mapping_alt_order.string());
235+
options_map["exclude-filter"].value() = boost::any(std::vector<std::string>{});
236+
237+
BOOST_CHECK_THROW(config_manager.initialize(options_map), Elements::Exception);
238+
}
220239

221240
//-----------------------------------------------------------------------------
222241

‎PhzExecutables/python/PhzExecutables/ExtractZList.py‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,10 @@ def defineSpecificProgramOptions():
5555
parser.add_argument('-mr','--max_record', type=int, default='-1',
5656
help='If>0 limit the number of object taken into account (default n=-1)')
5757
parser.add_argument('-of','--output_file', type=str, default='',
58-
help='Path to the file containing the redshift list (format based on the extention to be in [.fits,.csv,.dat]')
58+
help='Path to the file containing the redshift list (format based on the extention to be in [.fits,.csv,.dat,.conf]) if a *.conf is provided the list will be in a format to be used in Phosphoros CR')
59+
parser.add_argument('-cg','--config_group', type=str, default='<group>',
60+
help='if provided and the output file is a .conf, list the coma separated model groups to be configured (ex: <group1>,<group2>)')
61+
5962

6063
return parser
6164

@@ -95,7 +98,7 @@ def get_out_table(zs_extracted):
9598
#--------------------------
9699
def mainMethod(args):
97100
_, ext = os.path.splitext(args.output_file)
98-
if ext not in ['.fits','.dat','.csv']:
101+
if ext not in ['.fits','.dat','.csv','.conf']:
99102
raise ValueError(f'Unrecognised output file extention ({ext})')
100103
zs = extract_sub_sample(Table.read(args.input_catalog)[args.ref_z_col], args.skip_n_first, args.max_record)
101104
logger.info(f'Read {len(zs)} records from file {args.input_catalog} starting at record {args.skip_n_first}')
@@ -113,6 +116,14 @@ def mainMethod(args):
113116
out_t.write(args.output_file, format='fits', overwrite=True)
114117
elif ext == '.dat':
115118
out_t.write(args.output_file, format='ascii.no_header', overwrite=True)
119+
elif ext == '.conf':
120+
groups = args.config_group.split(',')
121+
122+
with open(args.output_file,"w") as f:
123+
for grp in groups:
124+
for z in zs_extracted:
125+
line = f'z-value-{grp}={z}\n'
126+
f.write(line)
116127
else:
117128
out_t.write(args.output_file, format='csv', overwrite=True)
118129

‎PhzExecutables/src/lib/BuildReferenceSample.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@
3737
#include "MathUtils/interpolation/interpolation.h"
3838
#include "XYDataset/CachedProvider.h"
3939

40+
#include <filesystem>
41+
4042
namespace Euclid {
4143
namespace PhzExecutables {
4244

@@ -122,10 +124,10 @@ void BuildReferenceSample::run(Euclid::Configuration::ConfigManager& config_mana
122124
if (boost::filesystem::exists(ref_sample_path)) {
123125
if (ref_sample_config.overwrite()) {
124126
std::vector<boost::filesystem::path> paths;
125-
for (auto const& entry : boost::filesystem::recursive_directory_iterator(ref_sample_path)) {
127+
for (auto const& entry : std::filesystem::recursive_directory_iterator(ref_sample_path.string())) {
126128

127129
if (entry.path().extension() == ".npy") {
128-
paths.emplace_back(entry);
130+
paths.emplace_back(boost::filesystem::path(entry.path().string()));
129131
}
130132
}
131133
logger.info() << "Clearing the Reference Sample dir of " << paths.size() << " *.npy files";

0 commit comments

Comments
 (0)