Skip to content

Commit 427b596

Browse files
bsipoczbsipocz
authored
Merge pull request #3313 from esdc-esac-esa-int/ESA_euclid_EUCLIDMNGT-1275_LE3_search_capability
Euclid: new method to retrieve scientific LE3 products
2 parents 8b57bef + d140a90 commit 427b596

File tree

6 files changed

+523
-6
lines changed

6 files changed

+523
-6
lines changed

‎CHANGES.rst

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,12 @@ alma
2424
- Bug fix in ``footprint_to_reg`` that did not allow regions to be plotted. [#3285]
2525

2626

27+
esa.euclid
28+
^^^^^^^^^^
29+
30+
- New method, get_scientific_product_list, to retrieve scientific LE3
31+
products. [#3313]
32+
2733
gaia
2834
^^^^
2935

@@ -82,10 +88,10 @@ mast
8288

8389
- Fix bug in ``utils.remove_duplicate_products`` that does not retain the order of the products in an input table. [#3314]
8490

85-
- Added ``return_uri_map`` parameter to ``Observations.get_cloud_uris`` to return a mapping of the input data product URIs
91+
- Added ``return_uri_map`` parameter to ``Observations.get_cloud_uris`` to return a mapping of the input data product URIs
8692
to the returned cloud URIs. [#3314]
8793

88-
- Added ``verbose`` parameter to ``Observations.get_cloud_uris`` to control whether warnings are logged when a product cannot
94+
- Added ``verbose`` parameter to ``Observations.get_cloud_uris`` to control whether warnings are logged when a product cannot
8995
be found in the cloud. [#3314]
9096

9197

‎astroquery/esa/euclid/__init__.py

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,70 @@ class Conf(_config.ConfigNamespace):
5959

6060
SIR_SCIENCE_FRAME_PRODUCTS = ['dpdSirScienceFrame']
6161

62+
VALID_LE3_PRODUCT_TYPES_CATEGORIES_GROUPS = {
63+
'Clusters of Galaxies': {
64+
'GrpCatalog': ['DpdLE3clAMICOModel', 'DpdLE3clDetMergeParams', 'DpdLE3clDetOnMockParams',
65+
'DpdLE3clDetInputParams', 'DpdLE3clAmicoAux', 'DpdLE3clAssociations', 'DpdLE3clPzwavAux',
66+
'DpdLE3clPZWAVDensity', 'DpdLE3clDetClusters', 'DpdLE3FullDet', 'DpdLE3clCatMergeParams',
67+
'DpdLE3clCATParams', 'DpdLE3clCcpInputParams', 'DpdLE3clRichMembers', 'DpdLE3clZClParams',
68+
'DpdLE3clGlueMatchParams', 'DpdLE3clMockGlueMatchParams'],
69+
'GrpClustering': ['DDpdLE3clPkDOA', 'DpdLE3clCovmatTwoPointCov2', 'DpdLE3clPkYam',
70+
'DpdLE3clTwoPointAutoCorrPol', 'DpolDpdLE3clCovmatPKCov1'],
71+
'GrpCOMB': ['DpdLE3clCombConfigurationSet', 'DpdLE3clCombCovMatDeltaSigmaCosmoDep',
72+
'DpdLE3clCombCovMatReducedShearCosmoDep', 'DpdLE3clCombCovMatReducedShearCosmoIndep',
73+
'DpdLE3clCombRedSheProf', 'DpdLE3clCombStackingCosmoDep', 'DpdLE3clCombStackingCosmoInd',
74+
'DpdLE3clCombUCovRedSheProf', 'DpdLE3clCombWLME'],
75+
'GrpLMF': ['DpdLE3clLMFOutput', 'DpdLE3clLMFParams'],
76+
'GrpSEL': ['DpdLE3clMatchForSelParams', 'DpdLE3clMockClusters', 'DpdLE3clRedshiftDistrib',
77+
'DpdLE3clSelRandom', 'DpdLE3clSelRandomParams', 'DpdLE3clSelSelFunc',
78+
'DpdLE3clSelSelFuncInputParams', 'DpdLE3clSelSinfoniaIniClMockInputParams',
79+
'DpdLE3clSelSinfoniaMockInputParams', 'DpdLE3clSelSubSample', 'DpdLE3clSinfoniaEllipticity'],
80+
'GrpTiling': ['DpdLE3clCLTile', 'DpdLE3clCommon', 'DpdLE3clConfigurationSet']},
81+
'External Data Products': {
82+
'LE3-ED configuration catalog': ['DpdLE3edConfigurationFile'],
83+
'LE3-ED match catalog': ['DpdLE3edMatchedCatalog']},
84+
'Galaxy Clustering Products': {
85+
'2PCF_PK': ['DpdLE3gcPkCross', 'DpdLE3gcPkDOA', 'DpdLE3gcPkYam', 'DpdLE3gcTwoPointAutoCorr',
86+
'DpdLE3gcTwoPointAutoCorrCart', 'DpdLE3gcTwoPointAutoCorrPol', 'DpdLE3gcTwoPointCrossCorr',
87+
'DpdLE3gcTwoPointCrossCorrCart', 'DpdLE3gcTwoPointCrossCorrPol', 'DpdLE3gcTwoPointRecAutoCorr',
88+
'DpdLE3gcTwoPointRecAutoCorrCart', 'DpdLE3gcTwoPointRecAutoCorrPol',
89+
'DpdLE3gcTwoPointRecCrossCorr', 'DpdLE3gcTwoPointRecCrossCorrCart',
90+
'DpdLE3gcTwoPointRecCrossCorrPol'],
91+
'3PCF_BK': ['DpdLE3gcBkMonopole', 'DpdLE3gcBkMultipole', 'DpdLE3gcThreePointAll', 'DpdLE3gcThreePointSin'],
92+
'CM-2PCF': ['DpdLE3gcCovmatTwoPointCov1D', 'DpdLE3gcCovmatTwoPointCov2Dcart',
93+
'DpdLE3gcCovmatTwoPointCov2Dpol', 'DpdLE3gcCovmatTwoPointCovMu',
94+
'DpdLE3gcCovmatTwoPointCovPro'],
95+
'CM-PK': ['DpdLE3gcCovmatPKCov1D', 'DpdLE3gcCovmatPKCov2Dcart', 'DpdLE3gcCovmatPKCov2Dpol']},
96+
'Internal Data Products': {
97+
'SEL Config/Stats': ['DpdLE3IDSELConfigurationSet', 'DpdLE3IDSELIDStatistics'],
98+
'SEL Wide Main': ['DpdLE3IDSELIDCatalog'],
99+
'SEL Wide': ['DpdLE3IDSELIDSubsampledCatalog'],
100+
'VMSP Group': ['DpdLE3IDVMSPConfiguration', 'DpdLE3IDVMSPDetectionModel', 'DpdLE3IDVMSPDistModel',
101+
'DpdLE3IDVMSPRandomCatalog']},
102+
'Weak Lensing Products': {
103+
'2D-MASS': ['DpdTwoDMassConvergenceClusters', 'DpdTwoDMassConvergencePatch',
104+
'DpdTwoDMassConvergencePatchesToSphere', 'DpdTwoDMassConvergenceSphere',
105+
'DpdTwoDMassParamsConvergenceClusters', 'DpdTwoDMassParamsConvergencePatch',
106+
'DpdTwoDMassParamsConvergencePatchesToSphere', 'DpdTwoDMassParamsConvergenceSphere',
107+
'DpdTwoDMassParamsPeakCatalogConvergence', 'DpdTwoDMassParamsPeakCatalogMassAperture',
108+
'DpdTwoDMassPeakCatalog'],
109+
'2PCF': ['DpdTwoPCFWLCOSEBIFilter', 'DpdTwoPCFWLParamsCOSEBIShearShear2D', 'DpdTwoPCFWLParamsClPosPos2D',
110+
'DpdTwoPCFWLParamsPEBPosShear2D', 'DpdTwoPCFWLParamsPEBShearShear2D', 'DpdTwoPCFWLParamsPosPos2D',
111+
'DpdTwoPCFWLParamsPosShear2D', 'DpdTwoPCFWLParamsShearShear2D', 'DpdTwoPCFWLCOSEBIShearShear2D',
112+
'DpdTwoPCFWLClPosPos2D', 'DpdTwoPCFWLPEBPosShear2D', 'DpdTwoPCFWLPEBShearShear2D',
113+
'DpdTwoPCFWLPosPos2D', 'DpdTwoPCFWLPosShear2D', 'DpdTwoPCFWLShearShear2D',
114+
'DpdCovarTwoPCFWLParams', 'DpdCovarTwoPCFWLClPosPos2D', 'DpdCovarTwoPCFWLCOSEBIShearShear2D',
115+
'DpdCovarTwoPCFWLPEBPosShear2D', 'DpdCovarTwoPCFWLPEBShearShear2D',
116+
'DpdCovarTwoPCFWLPosPos2D', 'DpdCovarTwoPCFWLPosShear2D', 'DpdCovarTwoPCFWLShearShear2D',
117+
'DpdCovarTwoPCFWLResampleCOSEBIShearShear2D', 'DpdCovarTwoPCFWLResampleClPosPos2D',
118+
'DpdCovarTwoPCFWLResamplePEBPosShear2D', 'DpdCovarTwoPCFWLResamplePEBShearShear2D',
119+
'DpdCovarTwoPCFWLResamplePosPos2D', 'DpdCovarTwoPCFWLResamplePosShear2D',
120+
'DpdCovarTwoPCFWLResampleShearShear2D'],
121+
'PK': ['DpdPKWLAlms', 'DpdPKWLCovMatrix2D', 'DpdPKWLEstimate2D', 'DpdPKWLMaps', 'DpdPKWLMixingMatrix2D']},
122+
'PHZ': {
123+
'PHZ': ['DpdBinMeanRedshift', 'DpdReferenceSample', 'DpdTomographicBins']}
124+
}
125+
62126
PRODUCT_TYPES = ['observation', 'mosaic']
63127

64128
SCHEMAS = ['sedm']

‎astroquery/esa/euclid/core.py

Lines changed: 168 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
"""
99
import binascii
1010
import os
11+
import pprint
1112
import tarfile
1213
import zipfile
1314
from collections.abc import Iterable
@@ -472,6 +473,10 @@ def login(self, *, user=None, password=None, credentials_file=None, verbose=Fals
472473
file containing user and password in two lines
473474
verbose : bool, optional, default 'False'
474475
flag to display information about the process
476+
477+
Returns
478+
-------
479+
None
475480
"""
476481
try:
477482
log.info(f"Login to Euclid TAP server: {self._TapPlus__getconnhandler().get_host_url()}")
@@ -503,6 +508,10 @@ def login_gui(self, verbose=False):
503508
----------
504509
verbose : bool, optional, default 'False'
505510
flag to display information about the process
511+
512+
Returns
513+
-------
514+
None
506515
"""
507516
try:
508517
log.info(f"Login to Euclid TAP server: {self._TapPlus__getconnhandler().get_host_url()}")
@@ -540,6 +549,11 @@ def logout(self, verbose=False):
540549
----------
541550
verbose : bool, optional, default 'False'
542551
flag to display information about the process
552+
553+
Returns
554+
-------
555+
None
556+
543557
"""
544558
try:
545559
super().logout(verbose=verbose)
@@ -779,7 +793,7 @@ def __get_tile_catalogue_list(self, *, tile_index, product_type, verbose=False):
779793
780794
Returns
781795
-------
782-
The list of products (astropy.table)
796+
The products in an astropy.table.Table
783797
"""
784798

785799
if tile_index is None:
@@ -884,7 +898,7 @@ def get_product_list(self, *, observation_id=None, tile_index=None, product_type
884898
#. NISP
885899
DpdNispRawFrame: NISP Raw Frame Product
886900
887-
#. Euclid LE2/LE3 products:
901+
#. Euclid LE2 products:
888902
889903
#. VIS
890904
DpdVisCalibratedQuadFrame: VIS Calibrated Frame Product
@@ -1276,5 +1290,157 @@ def get_datalinks(self, ids, *, linking_parameter='SOURCE_ID', verbose=False):
12761290

12771291
return self.__eucliddata.get_datalinks(ids=ids, linking_parameter=linking_parameter, verbose=verbose)
12781292

1293+
def get_scientific_product_list(self, *, observation_id=None, tile_index=None, category=None, group=None,
1294+
product_type=None, dataset_release='REGREPROC1_R2', verbose=False):
1295+
""" Gets the LE3 products (the high-level science data products).
1296+
1297+
Please note that not all combinations of category, group, and product_type are valid. Check the available values
1298+
in https://astroquery.readthedocs.io/en/latest/esa/euclid/euclid.html#appendix
1299+
1300+
Parameters
1301+
----------
1302+
observation_id: str, optional, default None.
1303+
It is not compatible with parameter tile_index.
1304+
tile_index: str, optional, default None.
1305+
It is not compatible with parameter observation_id.
1306+
category: str, optional, default None.
1307+
group : str, optional, default None
1308+
product_type : str, optional, default None
1309+
dataset_release : str, mandatory. Default REGREPROC1_R2
1310+
Data release from which data should be taken.
1311+
verbose : bool, optional, default 'False'
1312+
flag to display information about the process
1313+
1314+
Returns
1315+
-------
1316+
The products in an astropy.table.Table
1317+
1318+
"""
1319+
1320+
query_extra_condition = ""
1321+
1322+
if (observation_id is None and tile_index is None and category is None and group is None and product_type is
1323+
None):
1324+
raise ValueError("Include a valid parameter to retrieve a LE3 product.")
1325+
1326+
if dataset_release is None:
1327+
raise ValueError("The release is required.")
1328+
1329+
if observation_id is not None and tile_index is not None:
1330+
raise ValueError(self.__ERROR_MSG_REQUESTED_OBSERVATION_ID_AND_TILE_ID)
1331+
1332+
if tile_index is not None:
1333+
query_extra_condition = f" AND '{tile_index}' = ANY(tile_index_list) "
1334+
1335+
if observation_id is not None:
1336+
query_extra_condition = f" AND '{observation_id}' = ANY(observation_id_list) "
1337+
1338+
if category is not None:
1339+
1340+
try:
1341+
_ = conf.VALID_LE3_PRODUCT_TYPES_CATEGORIES_GROUPS[category]
1342+
except KeyError:
1343+
raise ValueError(
1344+
f"Invalid combination of parameters: category={category}. Valid values:\n "
1345+
f"{pprint.pformat(conf.VALID_LE3_PRODUCT_TYPES_CATEGORIES_GROUPS)}")
1346+
1347+
if group is not None:
1348+
1349+
try:
1350+
product_type_for_category_group_list = conf.VALID_LE3_PRODUCT_TYPES_CATEGORIES_GROUPS[category][
1351+
group]
1352+
except KeyError:
1353+
raise ValueError(
1354+
f"Invalid combination of parameters: category={category}; group={group}. Valid "
1355+
f"values:\n {pprint.pformat(conf.VALID_LE3_PRODUCT_TYPES_CATEGORIES_GROUPS)}")
1356+
1357+
if product_type is not None:
1358+
1359+
if product_type not in product_type_for_category_group_list:
1360+
raise ValueError(
1361+
f"Invalid combination of parameters: category={category}; group={group}; "
1362+
f"product_type={product_type}. Valid values:\n "
1363+
f"{pprint.pformat(conf.VALID_LE3_PRODUCT_TYPES_CATEGORIES_GROUPS)}")
1364+
1365+
query_extra_condition = query_extra_condition + f" AND product_type ='{product_type}' "
1366+
else:
1367+
1368+
final_products = ', '.join(f"'{w}'" for w in product_type_for_category_group_list)
1369+
query_extra_condition = query_extra_condition + f" AND product_type IN ({final_products}) "
1370+
else: # category is not None and group is None
1371+
1372+
product_type_for_category_group_list = [item for row in
1373+
conf.VALID_LE3_PRODUCT_TYPES_CATEGORIES_GROUPS[category]
1374+
.values() for item in row]
1375+
if product_type is not None:
1376+
1377+
if product_type not in product_type_for_category_group_list:
1378+
raise ValueError(
1379+
f"Invalid combination of parameters: category={category}; product_type={product_type}."
1380+
f" Valid values:\n {pprint.pformat(conf.VALID_LE3_PRODUCT_TYPES_CATEGORIES_GROUPS)}")
1381+
1382+
query_extra_condition = query_extra_condition + f" AND product_type = '{product_type}' "
1383+
1384+
else: # category is not None and group is None and product_type is None
1385+
final_products = ', '.join(f"'{w}'" for w in product_type_for_category_group_list)
1386+
query_extra_condition = query_extra_condition + f" AND product_type IN ({final_products}) "
1387+
else: # category is None
1388+
1389+
all_groups_dict = {}
1390+
for i in conf.VALID_LE3_PRODUCT_TYPES_CATEGORIES_GROUPS.keys():
1391+
all_groups_dict.update(conf.VALID_LE3_PRODUCT_TYPES_CATEGORIES_GROUPS[i])
1392+
1393+
if group is not None:
1394+
1395+
try:
1396+
_ = all_groups_dict[group]
1397+
except KeyError:
1398+
raise ValueError(
1399+
f"Invalid combination of parameters: group={group}. Valid values:\n "
1400+
f"{pprint.pformat(conf.VALID_LE3_PRODUCT_TYPES_CATEGORIES_GROUPS)}")
1401+
1402+
if product_type is not None:
1403+
1404+
if product_type not in all_groups_dict[group]:
1405+
raise ValueError(
1406+
f"Invalid combination of parameters: group={group}; product_type={product_type}. Valid "
1407+
f"values:\n {pprint.pformat(conf.VALID_LE3_PRODUCT_TYPES_CATEGORIES_GROUPS)}")
1408+
1409+
query_extra_condition = query_extra_condition + f" AND product_type = '{product_type}' "
1410+
else: # group is not None and product_type is None
1411+
1412+
product_type_for_group_list = all_groups_dict[group]
1413+
final_products = ', '.join(f"'{w}'" for w in product_type_for_group_list)
1414+
query_extra_condition = query_extra_condition + f" AND product_type IN ({final_products}) "
1415+
1416+
else: # category is None and group is None
1417+
1418+
product_type_for_category_group_list = [element for sublist in all_groups_dict.values() for element
1419+
in sublist]
1420+
1421+
if product_type is not None:
1422+
if product_type not in product_type_for_category_group_list:
1423+
raise ValueError(
1424+
f"Invalid combination of parameters: product_type={product_type}. Valid values:\n "
1425+
f"{pprint.pformat(conf.VALID_LE3_PRODUCT_TYPES_CATEGORIES_GROUPS)}")
1426+
1427+
query_extra_condition = query_extra_condition + f" AND product_type = '{product_type}' "
1428+
1429+
else:
1430+
query_extra_condition = query_extra_condition + ""
1431+
1432+
query = (
1433+
f"SELECT basic_download_data.basic_download_data_oid, basic_download_data.product_type, "
1434+
f"basic_download_data.product_id, CAST(basic_download_data.observation_id_list as text) AS "
1435+
f"observation_id_list, CAST(basic_download_data.tile_index_list as text) AS tile_index_list, "
1436+
f"CAST(basic_download_data.patch_id_list as text) AS patch_id_list, "
1437+
f"CAST(basic_download_data.filter_name as text) AS filter_name FROM sedm.basic_download_data WHERE "
1438+
f"release_name='{dataset_release}' {query_extra_condition} ORDER BY observation_id_list ASC")
1439+
1440+
job = super().launch_job(query=query, output_format='votable_plain', verbose=verbose,
1441+
format_with_results_compressed=('votable_gzip',))
1442+
1443+
return job.get_results()
1444+
12791445

12801446
Euclid = EuclidClass()

0 commit comments

Comments
 (0)