EHSTPCR-1113: review checks included

Author: jespinosaar

CHANGES.rst

@@ -31,7 +31,7 @@ esa.hubble
 
 - Update to TAP url to query data and download files, aligned with the new eHST Science Archive. [#2567][#2597]
 - Status and maintenance messages from eHST TAP when the module is instantiated. get_status_messages method to retrieve them. [#2597]
-- New methods to download single files and download FITS associated to an observation. [#2797]
+- New methods to download single files ``download_file`` and download FITS associated to an observation ``download_fits_files``. [#2797]
 - New function to retrieve all the files associated to an observation. [#2797]
 
 solarsystem.neodys
@@ -66,7 +66,7 @@ esa.hubble
   a lot faster. [#2524]
 - Method query_hst_tap has been deprecated and is replaced with query_tap, with the same arguments. [#2597]
 - Product types in download_product method have been modified to: PRODUCT, SCIENCE_PRODUCT or POSTCARD. [#2597]
-- query_criteria method now allows user to filter by Proposal ID. [#2797]
+- Added `proposal` keyword argument to several methods now allows to filter by Proposal ID. [#2797]
 
 alma
 ^^^^

astroquery/esa/hubble/core.py

@@ -28,6 +28,21 @@
 __all__ = ['ESAHubble', 'ESAHubbleClass']
 
 
+def check_rename_to_gz(filename):
+    rename = False
+    if os.path.exists(filename):
+        with open(filename, 'rb') as test_f:
+            if test_f.read(2) == b'\x1f\x8b' and not filename.endswith('.fits.gz'):
+                rename = True
+
+    if rename:
+        output = os.path.splitext(filename)[0] + '.fits.gz'
+        os.rename(filename, output)
+        return output
+    else:
+        return filename
+
+
 class ESAHubbleClass(BaseQuery):
     """
     Class to init ESA Hubble Module and communicate with eHST TAP
@@ -108,21 +123,7 @@ def download_product(self, observation_id, *, calibration_level=None,
         filename = self._get_product_filename(product_type, filename)
         self._tap.load_data(params_dict=params, output_file=filename, verbose=verbose)
 
-        return self.check_rename_to_gz(filename=filename)
-
-    def check_rename_to_gz(self, filename):
-        rename = False
-        if os.path.exists(filename):
-            with open(filename, 'rb') as test_f:
-                if test_f.read(2) == b'\x1f\x8b' and not filename.endswith('.fits.gz'):
-                    rename = True
-
-        if rename:
-            output = os.path.splitext(filename)[0] + '.fits.gz'
-            os.rename(filename, output)
-            return output
-        else:
-            return filename
+        return check_rename_to_gz(filename=filename)
 
     def __set_product_type(self, product_type):
         if product_type:
@@ -311,7 +312,7 @@ def download_fits_files(self, observation_id, *, verbose=False):
 
     def download_file(self, file, *, filename=None, verbose=False):
         """
-        Download a file from EHST based on its filename. Similar to get_a
+        Download a file from eHST based on its filename.
 
         Parameters
         ----------
@@ -335,7 +336,7 @@ def download_file(self, file, *, filename=None, verbose=False):
 
         self._tap.load_data(params_dict=params, output_file=filename, verbose=verbose)
 
-        return self.check_rename_to_gz(filename=filename)
+        return check_rename_to_gz(filename=filename)
 
     def get_postcard(self, observation_id, *, calibration_level="RAW",
                      resolution=256, filename=None, verbose=False):

astroquery/esa/hubble/tests/test_esa_hubble.py

@@ -23,6 +23,7 @@
 from requests.models import Response
 
 from astroquery.esa.hubble import ESAHubbleClass
+from astroquery.esa.hubble.core import check_rename_to_gz
 from astroquery.esa.hubble.tests.dummy_tap_handler import DummyHubbleTapHandler
 from astropy.utils.exceptions import AstropyDeprecationWarning
 
@@ -270,7 +271,7 @@ def test_download_fits(self, mock_associated_files):
     def test_is_not_gz(self, tmp_path):
         target_file = data_path('cone_search.vot')
         ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False)
-        assert ehst.check_rename_to_gz(target_file) == target_file
+        assert check_rename_to_gz(target_file) == target_file
 
     def test_is_gz(self, tmp_path):
         ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False)
@@ -281,7 +282,7 @@ def test_is_gz(self, tmp_path):
             f.write(b'')
         # with open(test_file, 'rb') as f_in, gzip.open(target_file, 'wb') as f_out:
         #     f_out.writelines(f_in)
-        assert ehst.check_rename_to_gz(target_file) == target_file + '.fits.gz'
+        assert check_rename_to_gz(target_file) == target_file + '.fits.gz'
 
     def test_get_columns(self):
         parameters = {'table_name': "table",

astroquery/esa/hubble/tests/test_esa_hubble_remote.py

@@ -59,7 +59,7 @@ def test_query_tap_async(self):
     def test_download_product(self):
         result = esa_hubble.query_tap(query=self.hst_query)
         observation_id = np.random.choice((result['observation_id']))
-        temp_file = self.temp_folder.name + "/" + observation_id
+        temp_file = os.path.join(self.temp_folder.name, observation_id)
         esa_hubble.download_product(observation_id=observation_id,
                                     filename=temp_file)
         possible_values = [os.path.exists(temp_file + '.jpg'),
@@ -71,7 +71,7 @@ def test_get_artifact(self):
         result = esa_hubble.query_tap(query=self.top_artifact_query)
         assert "artifact_id" in result.keys()
         artifact_id = np.random.choice(result["artifact_id"])
-        temp_file = self.temp_folder.name + "/" + artifact_id
+        temp_file = os.path.join(self.temp_folder.name, artifact_id)
         esa_hubble.get_artifact(artifact_id=artifact_id, filename=temp_file)
         possible_values = [os.path.exists(temp_file),
                            os.path.exists(temp_file + '.zip'),
@@ -81,7 +81,7 @@ def test_get_artifact(self):
     def test_cone_search(self):
         esa_hubble = ESAHubble()
         c = coordinates.SkyCoord("00h42m44.51s +41d16m08.45s", frame='icrs')
-        compressed_temp_file = self.temp_folder.name + "/cone_search_m31_5.vot.gz"
+        compressed_temp_file = os.path.join(self.temp_folder.name, "cone_search_m31_5.vot.gz")
         # open & extracting the file
         table = esa_hubble.cone_search(coordinates=c, radius=7, filename=compressed_temp_file, verbose=True)
         assert 'observation_id' in table.columns
@@ -120,6 +120,6 @@ def test_hst_simple_to_hap_simple(self):
         assert result == ['hst_16316_71_acs_sbc_f150lp_jec071i9']
 
     def test_query_target(self):
-        compressed_temp_file = self.temp_folder.name + "/" + "m31_query.xml.gz"
+        compressed_temp_file = os.path.join(self.temp_folder.name, "m31_query.xml.gz")
         table = esa_hubble.query_target(name="m3", filename=compressed_temp_file)
         assert 'observation_id' in table.columns

docs/esa/hubble/hubble.rst

@@ -19,10 +19,6 @@ in the EHST are synchronised with the MAST services for HST reprocessed
 public data and corresponding metadata.  Therefore, excluding proprietary
 data, all HST data in the EHST are identical to those in MAST.
 
-========
-Examples
-========
-
 It is highly recommended checking the status of eHST TAP before executing this module. To do this:
 
 .. doctest-remote-data::
@@ -34,6 +30,19 @@ It is highly recommended checking the status of eHST TAP before executing this m
 This method will retrieve the same warning messages shown in eHST Science Archive with information about
 service degradation.
 
+========
+Examples
+========
+
+.. note::
+    The recommended steps to work with eHST Astroquery module are described below:
+        #. Retrieve the desired observations, fulfilling the user requirements, using one of the following methods: ``query_target``, ``query_criteria``, ``cone_search`` or ``cone_search_criteria``. In the results, the user will allways find a column named 'observation_id' that will be used as a reference.
+        #. If all the products associated to an observation are required, then use ``download_product``.
+        #. If only FITS files associated to an observation are required, then use ``download_fits_files``.
+        #. It is possible to retrieve the name of the files associated to an observation using ``get_associated_files``, together with their calibration level, size and type.
+        #. Users can filter the previous list to get the specific files to download and use them in ``download_file`` function.
+        #. Use your algorithms and code to process the data.
+
 ----------------------------------------------
 1. Querying target names in the Hubble archive
 ----------------------------------------------
@@ -374,10 +383,12 @@ After retrieving the metadata, the user can filter the result table and get the
 The most important column is 'observation_id' and it is possible to use it to retrieve all the
 associated files.
 
-In eHST is it possible to download products based on their observation ID (mandatory) and
-a required calibration_level (RAW, CALIBRATED, PRODUCT or AUXILIARY) and/or product type (SCIENCE, PREVIEW, THUMBNAIL or AUXILIARY).
+.. note::
+    In eHST is it possible to download products based on their observation ID (mandatory) and
+    a required calibration_level (RAW, CALIBRATED, PRODUCT or AUXILIARY) and/or product type (SCIENCE, PREVIEW, THUMBNAIL or AUXILIARY).
 
-Deprecation Warning: product types PRODUCT, SCIENCE_PRODUCT or POSTCARD are no longer supported. Please modify your scripts accordingly.
+.. warning::
+    Deprecation Warning: product types PRODUCT, SCIENCE_PRODUCT or POSTCARD are no longer supported. Please modify your scripts accordingly.
 
 For instance, next commands will download all files for the raw calibration level
 of the observation 'j6fl25s4q' and it will store them in a file called
@@ -388,7 +399,8 @@ of the observation 'j6fl25s4q' and it will store them in a file called
   >>> from astroquery.esa.hubble import ESAHubble
   >>> esahubble = ESAHubble()
   >>> esahubble.download_product(observation_id="j6fl25s4q", calibration_level="RAW",
-  ...                            filename="raw_data_for_j6fl25s4q")  # doctest: +IGNORE_OUTPUT
+  ...                            filename="raw_data_for_j6fl25s4q")
+  'raw_data_for_j6fl25s4q.zip'
 
 This second example will download the science files associated to the observation 'j6fl25s4q' and it will store them in a file called
 'science_data_for_j6fl25s4q.zip', modifying the filename provided to ensure that the extension of the file is correct.
@@ -398,17 +410,20 @@ This second example will download the science files associated to the observatio
   >>> from astroquery.esa.hubble import ESAHubble
   >>> esahubble = ESAHubble()
   >>> esahubble.download_product(observation_id="j6fl25s4q", product_type="SCIENCE",
-  ...                            filename="science_data_for_j6fl25s4q")   # doctest: +IGNORE_OUTPUT
+  ...                            filename="science_data_for_j6fl25s4q")
+  'science_data_for_j6fl25s4q.zip'
 
 This third case will download the science files associated to the observation 'j6fl25s4q' in raw calibration level and it will store them in a file called
-'science_raw_data_for_j6fl25s4q.fits.gz', modifying the filename provided to ensure that the extension of the file is correct.
+'science_raw_data_for_j6fl25s4q.fits.gz', modifying the filename provided to ensure that the extension of the file is correct. There is only
+one file fulfilling these conditions and it is a FITS file, so the extension is adapted to the contents of the request.
 
 .. doctest-remote-data::
 
   >>> from astroquery.esa.hubble import ESAHubble
   >>> esahubble = ESAHubble()
   >>> esahubble.download_product(observation_id="j6fl25s4q", calibration_level="RAW",
-  ...                            filename="science_raw_data_for_j6fl25s4q", product_type="SCIENCE")   # doctest: +IGNORE_OUTPUT
+  ...                            filename="science_raw_data_for_j6fl25s4q", product_type="SCIENCE")
+  'science_raw_data_for_j6fl25s4q.fits.gz'
 
 If the user wants to filter the files to be downloaded, this module provides additional mechanisms.
 
@@ -448,7 +463,7 @@ In case the user is only interested in FITS files, this module contains a specif
 .. doctest-remote-data::
   >>> from astroquery.esa.hubble import ESAHubble
   >>> esahubble = ESAHubble()
-  >>> esahubble.download_fits_files(observation_id='w0ji0v01t')
+  >>> esahubble.download_fits_files(observation_id='w0ji0v01t') # doctest: +IGNORE_OUTPUT
 
 ---------------------------
 6. Getting Hubble postcards
@@ -458,7 +473,8 @@ In case the user is only interested in FITS files, this module contains a specif
 
   >>> from astroquery.esa.hubble import ESAHubble
   >>> esahubble = ESAHubble()
-  >>> esahubble.get_postcard(observation_id="j6fl25s4q", calibration_level="RAW", resolution=256, filename="raw_postcard_for_j6fl25s4q.jpg")  # doctest: +IGNORE_OUTPUT
+  >>> esahubble.get_postcard(observation_id="j6fl25s4q", calibration_level="RAW", resolution=256, filename="raw_postcard_for_j6fl25s4q.jpg")
+  'raw_postcard_for_j6fl25s4q.jpg'
 
 This will download the postcard for the observation 'J8VP03010' with low
 resolution (256) and it will stored in a jpg called
@@ -509,7 +525,7 @@ method returns the simple observations that make it up.
   >>> from astroquery.esa.hubble import ESAHubble
   >>> esahubble = ESAHubble()
   >>> result = esahubble.get_member_observations(observation_id="jdrz0c010")
-  >>> print(result)
+  >>> result
   ['jdrz0cjxq', 'jdrz0cjyq']
 
 
@@ -525,7 +541,7 @@ returns the corresponding HAP or HST observation
   >>> from astroquery.esa.hubble import ESAHubble
   >>> esahubble = ESAHubble()
   >>> result = esahubble.get_hap_hst_link(observation_id="hst_16316_71_acs_sbc_f150lp_jec071i9")
-  >>> print(result)
+  >>> result
   ['jec071i9q']