ESA HST module PyVO migration

CHANGES.rst

@@ -9,6 +9,11 @@ New Tools and Services
 API changes
 -----------
 
+esa.hubble
+^^^^^^^^^^
+
+- Migration to PyVO [#3367]
+
 gaia
 ^^^^
 

astroquery/esa/hubble/__init__.py

@@ -9,16 +9,23 @@
 
 """
 
+
 from astropy import config as _config
 from astropy.config import paths
 import os
 
+EHST_COMMON_SERVER = "https://hst.esac.esa.int/tap-server/"
+EHST_TAP_COMMON = "tap"
+
 
 class Conf(_config.ConfigNamespace):
     """
     Configuration parameters for `astroquery.esa.hubble`.
     """
-    EHST_TAP_SERVER = _config.ConfigItem("https://hst.esac.esa.int/tap-server/tap", "eHST TAP Server")
+    EHST_DOMAIN_SERVER = _config.ConfigItem(EHST_COMMON_SERVER, "eHST TAP Common Server")
+    EHST_TAP_SERVER = _config.ConfigItem(EHST_COMMON_SERVER + EHST_TAP_COMMON, "eHST TAP Server")
+    EHST_DATA_SERVER = _config.ConfigItem(EHST_COMMON_SERVER + 'data?', "eHST Data Server")
+    EHST_TABLES_SERVER = _config.ConfigItem(EHST_COMMON_SERVER + EHST_TAP_COMMON + "/tables", "eHST TAP Common Server")
     EHST_TARGET_ACTION = _config.ConfigItem("servlet/target-resolver?", "eHST Target Resolver")
     EHST_MESSAGES = _config.ConfigItem("notification?action=GetNotifications", "eHST Messages")
     TIMEOUT = 60

astroquery/esa/hubble/core.py

@@ -8,8 +8,9 @@
 European Space Agency (ESA)
 
 """
+
+
 import os
-from urllib.parse import urlencode
 
 import astroquery.esa.utils.utils as esautils
 
@@ -18,20 +19,20 @@
 from astropy.coordinates import Angle
 from numpy.ma import MaskedArray
 
-from astroquery.utils.tap import TapPlus
-from astroquery.query import BaseQuery
-import json
+from astroquery.query import BaseQuery, BaseVOQuery
 import warnings
+import pyvo
 from astropy.utils.exceptions import AstropyDeprecationWarning
 from astropy.utils.decorators import deprecated
 
 from . import conf
+import time
 from astroquery import log
 
 __all__ = ['ESAHubble', 'ESAHubbleClass']
 
 
-class ESAHubbleClass(BaseQuery):
+class ESAHubbleClass(BaseVOQuery, BaseQuery):
     """
     Class to init ESA Hubble Module and communicate with eHST TAP
     """
@@ -41,15 +42,33 @@
     product_types = ["SCIENCE", "PREVIEW", "THUMBNAIL", "AUXILIARY"]
     copying_string = "Copying file to {0}..."
 
-    def __init__(self, *, tap_handler=None, show_messages=True):
-        if tap_handler is None:
-            self._tap = TapPlus(url=conf.EHST_TAP_SERVER,
-                                data_context='data', client_id="ASTROQUERY")
+    def __init__(self, *, show_messages=True, auth_session=None):
+        super().__init__()
+
+        self._vo = None
+        # Checks if auth session has been defined. If not, create a new session
+        if auth_session:
+            self._auth_session = auth_session
         else:
-            self._tap = tap_handler
+            self._auth_session = esautils.ESAAuthSession()
+            self._auth_session.add_security_method_for_url(url=conf.EHST_TAP_SERVER, security_method="anonymous")
+            self._auth_session.add_security_method_for_url(
+                url=conf.EHST_TABLES_SERVER,
+                security_method="anonymous",
+                exact=True
+            )
+
         if show_messages:
             self.get_status_messages()
 
+    @property
+    def vo(self) -> pyvo.dal.TAPService:
+        if self._vo is None:
+            self._vo = pyvo.dal.TAPService(
+                conf.EHST_TAP_SERVER, session=self._auth_session)
+
+        return self._vo
+
     def download_product(self, observation_id, *, calibration_level=None,
                          filename=None, folder=None, verbose=False, product_type=None):
         """
@@ -95,7 +114,6 @@
                 AstropyDeprecationWarning)
 
         params = {"OBSERVATIONID": observation_id,
-                  "TAPCLIENT": "ASTROQUERY",
                   "RETRIEVAL_TYPE": "OBSERVATION"}
 
         if filename is None:
@@ -112,8 +130,13 @@
 
         filename = self._get_product_filename(product_type, filename)
         output_file = self.__get_download_path(folder, filename)
-        self._tap.load_data(params_dict=params, output_file=output_file, verbose=verbose)
-
+        esautils.download_file(
+            url=conf.EHST_DATA_SERVER,
+            session=self.vo._session,
+            params=params,
+            verbose=verbose,
+            filename=output_file
+        )
         return esautils.check_rename_to_gz(filename=output_file)
 
     def __get_download_path(self, folder, filename):
@@ -417,12 +440,17 @@
         None. The file is associated
         """
 
-        params = {"RETRIEVAL_TYPE": "PRODUCT", "ARTIFACTID": file, "TAPCLIENT": "ASTROQUERY"}
+        params = {"RETRIEVAL_TYPE": "PRODUCT", "ARTIFACTID": file}
         if filename is None:
             filename = file
         output_file = self.__get_download_path(folder, filename)
-        self._tap.load_data(params_dict=params, output_file=output_file, verbose=verbose)
-
+        esautils.download_file(
+            url=conf.EHST_DATA_SERVER,
+            session=self.vo._session,
+            params=params,
+            verbose=verbose,
+            filename=output_file
+        )
         return esautils.check_rename_to_gz(filename=output_file)
 
     def get_postcard(self, observation_id, *, calibration_level="RAW",
@@ -458,8 +486,7 @@
 
         params = {"RETRIEVAL_TYPE": "OBSERVATION",
                   "OBSERVATIONID": observation_id,
-                  "PRODUCTTYPE": "PREVIEW",
-                  "TAPCLIENT": "ASTROQUERY"}
+                  "PRODUCTTYPE": "PREVIEW"}
         if calibration_level:
             params["CALIBRATIONLEVEL"] = calibration_level
 
@@ -469,8 +496,8 @@
         if filename is None:
             filename = observation_id
 
-        self._tap.load_data(params_dict=params, output_file=filename, verbose=verbose)
-
+        esautils.download_file(url=conf.EHST_DATA_SERVER, session=self.vo._session, params=params, verbose=verbose,
+                               filename=filename)
         return filename
 
     def __get_product_type_by_resolution(self, resolution):
@@ -670,18 +697,17 @@
         try:
             params = {"TARGET_NAME": target,
                       "RESOLVER_TYPE": "ALL",
-                      "FORMAT": "json",
-                      "TAPCLIENT": "ASTROQUERY"}
-
-            subContext = conf.EHST_TARGET_ACTION
-            connHandler = self._tap._TapPlus__getconnhandler()
-            data = urlencode(params)
-            target_response = connHandler.execute_secure(subContext, data, verbose=True)
-            for line in target_response:
-                target_result = json.loads(line.decode("utf-8"))
-                if target_result['objects']:
-                    ra = target_result['objects'][0]['raDegrees']
-                    dec = target_result['objects'][0]['decDegrees']
+                      "FORMAT": "json"}
+
+            target_response = esautils.execute_servlet_request(
+                tap=self.vo,
+                query_params=params,
+                url=conf.EHST_DOMAIN_SERVER + conf.EHST_TARGET_ACTION
+            )
+            if target_response:
+                if target_response['objects']:
+                    ra = target_response['objects'][0]['raDegrees']
+                    dec = target_response['objects'][0]['decDegrees']
             return SkyCoord(ra=ra, dec=dec, unit="deg")
         except (ValueError, KeyError):
             raise ValueError("This target cannot be resolved")
@@ -747,18 +773,18 @@
                 A table object
                 """
         if async_job:
-            job = self._tap.launch_job_async(query=query,
-                                             output_file=output_file,
-                                             output_format=output_format,
-                                             verbose=verbose,
-                                             dump_to_file=output_file is not None)
+            job = self.vo.submit_job(query)
+            job.run()
+            while job.phase == 'EXECUTING':
+                time.sleep(3)
+            result = job.fetch_result().to_table()
         else:
-            job = self._tap.launch_job(query=query, output_file=output_file,
-                                       output_format=output_format,
-                                       verbose=verbose,
-                                       dump_to_file=output_file is not None)
-        table = job.get_results()
-        return table
+            result = self.vo.search(query).to_table()
+
+        if output_file:
+            esautils.download_table(result, output_file, output_format)
+
+        return result
 
     @deprecated(since="0.4.7", alternative="query_tap")
     def query_hst_tap(self, query, *, async_job=False, output_file=None,
@@ -923,34 +949,35 @@
         -------
         A list of tables
         """
-
-        tables = self._tap.load_tables(only_names=only_names,
-                                       include_shared_tables=False,
-                                       verbose=verbose)
-        if only_names is True:
-            table_names = []
-            for t in tables:
-                table_names.append(t.name)
-            return table_names
+        tables = self.vo.tables
+        if only_names:
+            return list(tables.keys())
         else:
-            return tables
+            return list(tables.values())
 
     def get_status_messages(self):
         """Retrieve the messages to inform users about
         the status of eHST TAP
         """
 
         try:
-            subContext = conf.EHST_MESSAGES
-            connHandler = self._tap._TapPlus__getconnhandler()
-            response = connHandler.execute_tapget(subContext, verbose=False)
-            if response.status == 200:
-                for line in response:
-                    string_message = line.decode("utf-8")
-                    print(string_message[string_message.index('=') + 1:])
+            esautils.execute_servlet_request(
+                url=conf.EHST_TAP_SERVER + "/" + conf.EHST_MESSAGES,
+                tap=self.vo,
+                query_params={},
+                parser_method=self.parse_messages_response
+            )
         except OSError:
             print("Status messages could not be retrieved")
 
+    def parse_messages_response(self, response):
+        string_messages = []
+        for line in response.iter_lines():
+            string_message = line.decode("utf-8")
+            string_messages.append(string_message[string_message.index('=') + 1:])
+            print(string_messages[len(string_messages)-1])
+        return string_messages
+
     def get_columns(self, table_name, *, only_names=True, verbose=False):
         """Get the available columns for a table in EHST TAP service
 
@@ -968,9 +995,7 @@
         A list of columns
         """
 
-        tables = self._tap.load_tables(only_names=False,
-                                       include_shared_tables=False,
-                                       verbose=verbose)
+        tables = self.get_tables(only_names=False)
         columns = None
         for t in tables:
             if str(t.name) == str(table_name):
@@ -1061,4 +1086,5 @@
         return full_path
 
 
-ESAHubble = ESAHubbleClass()
+# Neet to be false in order to avoid errors in tests
+ESAHubble = ESAHubbleClass(show_messages=False)