Move logic into get_cloud_uris()

Author: snbianco

CHANGES.rst

@@ -169,8 +169,7 @@ mast
 
 - Fix bug in ``Catalogs.query_criteria()`` to use ``page`` and ``pagesize`` parameters correctly. [#3065]
 
-- Add ``mast.Observations.get_cloud_uris_query`` method so that given a set of query criteria and optional filters,
-  the user receives a list of cloud data URIs for matching data products. [#3064]
+- Modify ``mast.Observations.get_cloud_uris`` to also accept query criteria and data product filters. [#3064]
 
 
 0.4.7 (2024-03-08)

astroquery/mast/observations.py

@@ -774,26 +774,56 @@ def download_products(self, products, *, download_dir=None, flat=False,
 
         return manifest
 
-    def get_cloud_uris(self, data_products, *, include_bucket=True, full_url=False):
+    def get_cloud_uris(self, data_products=None, *, include_bucket=True, full_url=False, pagesize=None, page=None,
+                       mrp_only=False, extension=None, filter_products={}, **criteria):
         """
-        Takes an `~astropy.table.Table` of data products and returns the associated cloud data uris.
+        Given an `~astropy.table.Table` of data products or query criteria and filter parameters,
+        returns the associated cloud data URIs.
 
         Parameters
         ----------
         data_products : `~astropy.table.Table`
-            Table containing products to be converted into cloud data uris.
+            Table containing products to be converted into cloud data uris. If provided, this will supercede
+            page_size, page, or any arguments passed in as **criteria.
         include_bucket : bool
-            Default True. When false returns the path of the file relative to the
+            Default True. When False, returns the path of the file relative to the
             top level cloud storage location.
             Must be set to False when using the full_url argument.
         full_url : bool
             Default False. Return an HTTP fetchable url instead of a cloud uri.
             Must set include_bucket to False to use this option.
+        pagesize : int, optional
+            Default None. Can be used to override the default pagesize when making a query.
+            E.g. when using a slow internet connection. Query criteria must also be provided.
+        page : int, optional
+            Default None. Can be used to override the default behavior of all results being returned for a query
+            to obtain one specific page of results. Query criteria must also be provided.
+        mrp_only : bool, optional
+            Default False. When set to True, only "Minimum Recommended Products" will be returned.
+        extension : string or array, optional
+            Default None. Option to filter by file extension.
+        filter_products : dict, optional
+            Filters to be applied to data products.  Valid filters are all products fields listed
+            `here <https://masttest.stsci.edu/api/v0/_productsfields.html>`__.
+            The column name as a string is the key. The corresponding value is one
+            or more acceptable values for that parameter.
+            Filter behavior is AND between the filters and OR within a filter set.
+            For example: {"productType": "SCIENCE", "extension"=["fits","jpg"]}
+        **criteria
+            Criteria to apply. At least one non-positional criteria must be supplied.
+            Valid criteria are coordinates, objectname, radius (as in `query_region` and `query_object`),
+            and all observation fields returned by the ``get_metadata("observations")``.
+            The Column Name is the keyword, with the argument being one or more acceptable values for that parameter,
+            except for fields with a float datatype where the argument should be in the form [minVal, maxVal].
+            For non-float type criteria wildcards maybe used (both * and % are considered wildcards), however
+            only one wildcarded value can be processed per criterion.
+            RA and Dec must be given in decimal degrees, and datetimes in MJD.
+            For example: filters=["FUV","NUV"],proposal_pi="Ost*",t_max=[52264.4586,54452.8914]
 
         Returns
         -------
         response : list
-            List of URIs generated from the data products, list way contain entries that are None
+            List of URIs generated from the data products. May contain entries that are None
             if data_products includes products not found in the cloud.
         """
 
@@ -802,6 +832,29 @@ def get_cloud_uris(self, data_products, *, include_bucket=True, full_url=False):
                 'Please enable anonymous cloud access by calling `enable_cloud_dataset` method. '
                 'Refer to `~astroquery.mast.ObservationsClass.enable_cloud_dataset` documentation for more info.')
 
+        if data_products is None:
+            if not criteria:
+                raise InvalidQueryError(
+                    'Please provide either a `~astropy.table.Table` of data products or query criteria.'
+                )
+            else:
+                # Get table of observations based on query criteria
+                obs = self.query_criteria(pagesize=pagesize, page=page, **criteria)
+
+                if not len(obs):
+                    # Warning raised by ~astroquery.mast.ObservationsClass.query_criteria
+                    return
+
+                # Return list of associated data products
+                data_products = self.get_product_list(obs)
+
+        # Filter product list
+        data_products = self.filter_products(data_products, mrp_only=mrp_only, extension=extension, **filter_products)
+
+        if not len(data_products):
+            warnings.warn("No matching products to fetch associated cloud URIs.", NoResultsWarning)
+            return
+
         # Remove duplicate products
         data_products = self._remove_duplicate_products(data_products)
 
@@ -841,59 +894,6 @@ def get_cloud_uri(self, data_product, *, include_bucket=True, full_url=False):
         # Query for product URIs
         return self._cloud_connection.get_cloud_uri(data_product, include_bucket, full_url)
 
-    def get_cloud_uris_query(self, *, pagesize=None, page=None, mrp_only=False, extension=None,
-                             filter_products={}, **criteria):
-        """
-        Given a set of criteria and optional filters, get a list of matching data products and return their
-        associated cloud data URIs.
-
-        Parameters
-        ----------
-        pagesize : int, optional
-            Default None. Can be used to override the default pagesize.
-            E.g. when using a slow internet connection.
-        page : int, optional
-            Default None. Can be used to override the default behavior of all results being returned to obtain
-            one specific page of results.
-        mrp_only : bool, optional
-            Default False. When set to True, only "Minimum Recommended Products" will be returned.
-        extension : string or array, optional
-            Default None. Option to filter by file extension.
-        filter_products : dict, optional
-            Filters to be applied.  Valid filters are all products fields listed
-            `here <https://masttest.stsci.edu/api/v0/_productsfields.html>`__.
-            The column name as a string is the key. The corresponding value is one
-            or more acceptable values for that parameter.
-            Filter behavior is AND between the filters and OR within a filter set.
-            For example: {"productType": "SCIENCE", "extension"=["fits","jpg"]}
-        **criteria
-            Criteria to apply. At least one non-positional criteria must be supplied.
-            Valid criteria are coordinates, objectname, radius (as in `query_region` and `query_object`),
-            and all observation fields returned by the ``get_metadata("observations")``.
-            The Column Name is the keyword, with the argument being one or more acceptable values for that parameter,
-            except for fields with a float datatype where the argument should be in the form [minVal, maxVal].
-            For non-float type criteria wildcards maybe used (both * and % are considered wildcards), however
-            only one wildcarded value can be processed per criterion.
-            RA and Dec must be given in decimal degrees, and datetimes in MJD.
-            For example: filters=["FUV","NUV"],proposal_pi="Ost*",t_max=[52264.4586,54452.8914]
-
-        Returns
-        -------
-        response : list
-
-        """
-        # Get table of observations based on query criteria
-        obs = self.query_criteria(pagesize=pagesize, page=page, **criteria)
-
-        # Return list of associated data products
-        prod = self.get_product_list(obs)
-
-        # Filter product list
-        filt = self.filter_products(prod, mrp_only=mrp_only, extension=extension, **filter_products)
-
-        # Return list of cloud URIs
-        return self.get_cloud_uris(filt)
-
     def _remove_duplicate_products(self, data_products):
         """
         Removes duplicate data products that have the same dataURI.

astroquery/mast/tests/test_mast_remote.py

@@ -544,6 +544,11 @@ def test_get_cloud_uris(self):
 
         assert len(uris) > 0, f'Products for OBSID {test_obs_id} were not found in the cloud.'
 
+        # check for warning if no data products match filters
+        with pytest.warns(NoResultsWarning):
+            Observations.get_cloud_uris(products,
+                                        extension='png')
+
     def test_get_cloud_uris_query(self):
         pytest.importorskip("boto3")
 
@@ -559,12 +564,20 @@ def test_get_cloud_uris_query(self):
         s3_uris = Observations.get_cloud_uris(filt)
 
         # get uris with streamlined function
-        uris = Observations.get_cloud_uris_query(target_name=234295610,
-                                                 provenance_name="SPOC",
-                                                 sequence_number=[1, 2],
-                                                 filter_products={'calib_level': [2]})
+        uris = Observations.get_cloud_uris(target_name=234295610,
+                                           provenance_name="SPOC",
+                                           sequence_number=[1, 2],
+                                           filter_products={'calib_level': [2]})
         assert s3_uris == uris
 
+        # check that InvalidQueryError is thrown if neither data_products or **criteria are defined
+        with pytest.raises(InvalidQueryError):
+            Observations.get_cloud_uris(filter_products={'calib_level': [2]})
+
+        # check for warning if query returns no observations
+        with pytest.warns(NoResultsWarning):
+            Observations.get_cloud_uris(target_name=234295611)
+
     ######################
     # CatalogClass tests #
     ######################

docs/mast/mast_obsquery.rst

@@ -427,8 +427,9 @@ MAST until it is disabled with `~astroquery.mast.ObservationsClass.disable_cloud
 To directly access a list of cloud URIs for a given dataset, use the
 `~astroquery.mast.ObservationsClass.get_cloud_uris`
 function (Python will prompt you to enable cloud access if you haven't already).
-To return a list of cloud URIs based on query criteria and product filters, use the
-`~astroquery.mast.ObservationsClass.get_cloud_uris_query` function.
+With this function, users may specify a `~astropy.table.Table` of data products or 
+query criteria. Query criteria are supplied as keyword arguments, and product filters 
+may be supplied through the ``mrp_only``, ``extension``, and ``filter_products`` parameters.
 
 When cloud access is enabled, the standard download function
 `~astroquery.mast.ObservationsClass.download_products` preferentially pulls files from AWS when they
@@ -462,8 +463,9 @@ To get a list of S3 URIs, use the following workflow:
    ...
    >>> Observations.disable_cloud_dataset()
 
-Alternatively, you can use the streamlined `~astroquery.mast.ObservationsClass.get_cloud_uris_query` function. This approach is recommended
-for code brevity. Query criteria are supplied as keyword arguments, and filters are supplied through the ``filter_products`` parameter.
+Alternatively, this workflow can be streamlined by providing the query criteria directly to `~astroquery.mast.ObservationsClass.get_cloud_uris`.
+This approach is recommended for code brevity. Query criteria are supplied as keyword arguments, and filters are supplied through the 
+``filter_products`` parameter. If both ``data_products`` and query criteria are provided, ``data_products`` takes precedence.
 
 .. doctest-skip::
 
@@ -474,12 +476,12 @@ for code brevity. Query criteria are supplied as keyword arguments, and filters
    INFO: Using the S3 STScI public dataset [astroquery.mast.core]
    ...
    >>> # Getting the cloud URIs
-   >>> s3_uris = Observations.get_cloud_uris_query(obs_collection='HST',
-                                                   filters='F606W',
-                                                   instrument_name='ACS/WFC',
-                                                   proposal_id=['12062'],
-                                                   dataRights='PUBLIC',
-                                                   filter_products={'productSubGroupDescription': 'DRZ'})
+   >>> s3_uris = Observations.get_cloud_uris(obs_collection='HST',
+                                             filters='F606W',
+                                             instrument_name='ACS/WFC',
+                                             proposal_id=['12062'],
+                                             dataRights='PUBLIC',
+                                             filter_products={'productSubGroupDescription': 'DRZ'})
    >>> print(s3_uris)
    ['s3://stpubdata/hst/public/jbev/jbeveo010/jbeveo010_drz.fits', 's3://stpubdata/hst/public/jbev/jbevet010/jbevet010_drz.fits']
    ...