Merge pull request #2376 from esdc-esac-esa-int/gaia-astroquery-1.1

Gaia astroquery 1.1

Author: bsipocz

CHANGES.rst

@@ -4,12 +4,19 @@
 New Tools and Services
 ----------------------
 
+gaia
+^^^^
+
+- TAP notifications service is now available for Gaia. If there is notification for the users,
+  for example planned or our unplanned downtimes of the archive, etc. The notification
+  will be also visible when accessing the archive through Astroquery. [#2376]
 
 hsa
 ^^^
 
 - New module to access ESA Herschel mission. [#2122]
 
+
 Service fixes and enhancements
 ------------------------------
 
@@ -49,6 +56,13 @@ oac
 - Fix bug in parsing events that contain html tags (e.g. in their alias
   field). [#2423]
 
+gaia
+^^^^
+
+- Method 'load_data' now has the parameter 'valid_data' set to False by default.
+  With this change the epoch photometry service returns all data associated
+  to a given source. [#2376]
+
 
 Infrastructure, Utility and Other Changes and Additions
 -------------------------------------------------------

astroquery/gaia/core.py

@@ -33,6 +33,7 @@
 from astropy import units as u
 import warnings
 from astroquery.exceptions import InputWarning
+from collections.abc import Iterable
 
 
 class GaiaClass(TapPlus):
@@ -44,14 +45,15 @@ class GaiaClass(TapPlus):
     MAIN_GAIA_TABLE_DEC = conf.MAIN_GAIA_TABLE_DEC
     ROW_LIMIT = conf.ROW_LIMIT
     VALID_DATALINK_RETRIEVAL_TYPES = conf.VALID_DATALINK_RETRIEVAL_TYPES
+    GAIA_MESSAGES = "notification?action=GetNotifications"
 
     def __init__(self, tap_plus_conn_handler=None,
                  datalink_handler=None,
                  gaia_tap_server='https://gea.esac.esa.int/',
                  gaia_data_server='https://gea.esac.esa.int/',
                  tap_server_context="tap-server",
                  data_server_context="data-server",
-                 verbose=False):
+                 verbose=False, show_server_messages=True):
         super(GaiaClass, self).__init__(url=gaia_tap_server,
                                         server_context=tap_server_context,
                                         tap_context="tap",
@@ -74,6 +76,10 @@ def __init__(self, tap_plus_conn_handler=None,
         else:
             self.__gaiadata = datalink_handler
 
+        # Enable notifications
+        if show_server_messages:
+            self.get_status_messages()
+
     def login(self, user=None, password=None, credentials_file=None,
               verbose=False):
         """Performs a login.
@@ -158,7 +164,7 @@ def logout(self, verbose=False):
         except HTTPError as err:
             log.error("Error logging out data server")
 
-    def load_data(self, ids, data_release=None, data_structure='INDIVIDUAL', retrieval_type="ALL", valid_data=True,
+    def load_data(self, ids, data_release=None, data_structure='INDIVIDUAL', retrieval_type="ALL", valid_data=False,
                   band=None, avoid_datatype_check=False, format="votable", output_file=None,
                   overwrite_output_file=False, verbose=False):
         """Loads the specified table
@@ -185,12 +191,12 @@ def load_data(self, ids, data_release=None, data_structure='INDIVIDUAL', retriev
             retrieval type identifier. For GAIA DR2 possible values are ['EPOCH_PHOTOMETRY']
             For future GAIA DR3 (Once published), possible values will be ['EPOC_PHOTOMETRY', 'RVS', 'XP_CONTINUOUS',
             'XP_SAMPLED', 'MCMC_GSPPHOT' or 'MCMC_MSC']
-        valid_data : bool, optional, default True
-            By default, the epoch photometry service returns only valid data,
-            that is, all data rows where flux is not null and
-            rejected_by_photometry flag is not true. In order to retrieve
-            all data associated to a given source without this filter,
-            this request parameter should be included (valid_data=False)
+        valid_data : bool, optional, default False
+            By default, the epoch photometry service returns all available data, including
+            data rows where flux is null and/or the rejected_by_photometry flag is set to True.
+            In order to retrieve only valid data (data rows where flux is not null and/or the
+            rejected_by_photometry flag is set to False) this request parameter should be included
+            with valid_data=True.
         band : str, optional, default None, valid values: G, BP, RP
             By default, the epoch photometry service returns all the
             available photometry bands for the requested source.
@@ -911,5 +917,27 @@ def launch_job_async(self, query, name=None, output_file=None,
                                         upload_table_name=upload_table_name,
                                         autorun=autorun)
 
+    def get_status_messages(self):
+        """Retrieve the messages to inform users about
+        the status of Gaia TAP
+        """
+        try:
+            subContext = self.GAIA_MESSAGES
+            connHandler = self._TapPlus__getconnhandler()
+            response = connHandler.execute_tapget(subContext, False)
+            if response.status == 200:
+                if isinstance(response, Iterable):
+                    for line in response:
+
+                        try:
+                            print(line.decode("utf-8").split('=', 1)[1])
+                        except ValueError as e:
+                            print(e)
+                        except IndexError:
+                            print("Archive down for maintenance")
+
+        except OSError:
+            print("Status messages could not be retrieved")
+
 
 Gaia = GaiaClass()

astroquery/gaia/tests/test_gaiatap.py

@@ -41,10 +41,50 @@ def data_path(filename):
 
 class TestTap:
 
+    def test_show_message(self):
+        connHandler = DummyConnHandler()
+
+        dummy_response = DummyResponse()
+        dummy_response.set_status_code(200)
+        dummy_response.set_message("OK")
+
+        message_text = "1653401204784D[type: -100,-1]=Gaia dev is under maintenance"
+
+        dummy_response.set_data(method='GET',
+                                context=None,
+                                body=message_text,
+                                headers=None)
+        connHandler.set_default_response(dummy_response)
+
+        # show_server_messages
+        tableRequest = 'notification?action=GetNotifications'
+        connHandler.set_response(tableRequest, dummy_response)
+
+        tapplus = TapPlus("http://test:1111/tap", connhandler=connHandler)
+        tap = GaiaClass(connHandler, tapplus, show_server_messages=True)
+
     def test_query_object(self):
         conn_handler = DummyConnHandler()
+        # Launch response: we use default response because the query contains
+        # decimals
+        dummy_response = DummyResponse()
+        dummy_response.set_status_code(200)
+        dummy_response.set_message("OK")
+
+        message_text = "1653401204784D[type: -100,-1]=Gaia dev is under maintenance"
+
+        dummy_response.set_data(method='GET',
+                                context=None,
+                                body=message_text,
+                                headers=None)
+        conn_handler.set_default_response(dummy_response)
+
+        # show_server_messages
+        tableRequest = 'notification?action=GetNotifications'
+        conn_handler.set_response(tableRequest, dummy_response)
+
         tapplus = TapPlus("http://test:1111/tap", connhandler=conn_handler)
-        tap = GaiaClass(conn_handler, tapplus)
+        tap = GaiaClass(conn_handler, tapplus, show_server_messages=True)
         # Launch response: we use default response because the query contains
         # decimals
         response_launch_job = DummyResponse()
@@ -125,7 +165,7 @@ def test_query_object(self):
     def test_query_object_async(self):
         conn_handler = DummyConnHandler()
         tapplus = TapPlus("http://test:1111/tap", connhandler=conn_handler)
-        tap = GaiaClass(conn_handler, tapplus)
+        tap = GaiaClass(conn_handler, tapplus, show_server_messages=False)
         jobid = '12345'
         # Launch response
         response_launch_job = DummyResponse()
@@ -220,7 +260,7 @@ def test_query_object_async(self):
     def test_cone_search_sync(self):
         conn_handler = DummyConnHandler()
         tapplus = TapPlus("http://test:1111/tap", connhandler=conn_handler)
-        tap = GaiaClass(conn_handler, tapplus)
+        tap = GaiaClass(conn_handler, tapplus, show_server_messages=False)
         # Launch response: we use default response because the query contains
         # decimals
         response_launch_job = DummyResponse()
@@ -273,7 +313,7 @@ def test_cone_search_sync(self):
     def test_cone_search_async(self):
         conn_handler = DummyConnHandler()
         tapplus = TapPlus("http://test:1111/tap", connhandler=conn_handler)
-        tap = GaiaClass(conn_handler, tapplus)
+        tap = GaiaClass(conn_handler, tapplus, show_server_messages=False)
         jobid = '12345'
         # Launch response
         response_launch_job = DummyResponse()
@@ -380,7 +420,7 @@ def __check_results_column(self, results, column_name, description, unit,
 
     def test_load_data(self):
         dummy_handler = DummyTapHandler()
-        tap = GaiaClass(dummy_handler, dummy_handler)
+        tap = GaiaClass(dummy_handler, dummy_handler, show_server_messages=False)
 
         ids = "1,2,3,4"
         retrieval_type = "epoch_photometry"
@@ -419,7 +459,7 @@ def test_load_data(self):
 
     def test_get_datalinks(self):
         dummy_handler = DummyTapHandler()
-        tap = GaiaClass(dummy_handler, dummy_handler)
+        tap = GaiaClass(dummy_handler, dummy_handler, show_server_messages=False)
         ids = ["1", "2", "3", "4"]
         verbose = True
         parameters = {}
@@ -431,7 +471,7 @@ def test_get_datalinks(self):
     def test_xmatch(self):
         conn_handler = DummyConnHandler()
         tapplus = TapPlus("http://test:1111/tap", connhandler=conn_handler)
-        tap = GaiaClass(conn_handler, tapplus)
+        tap = GaiaClass(conn_handler, tapplus, show_server_messages=False)
         jobid = '12345'
         # Launch response
         response_launch_job = DummyResponse()
@@ -579,7 +619,7 @@ def test_xmatch(self):
     def test_login(self, mock_login):
         conn_handler = DummyConnHandler()
         tapplus = TapPlus("http://test:1111/tap", connhandler=conn_handler)
-        tap = GaiaClass(conn_handler, tapplus)
+        tap = GaiaClass(conn_handler, tapplus, show_server_messages=False)
         tap.login("user", "password")
         assert (mock_login.call_count == 2)
         mock_login.side_effect = HTTPError("Login error")
@@ -591,7 +631,7 @@ def test_login(self, mock_login):
     def test_login_gui(self, mock_login_gui, mock_login):
         conn_handler = DummyConnHandler()
         tapplus = TapPlus("http://test:1111/tap", connhandler=conn_handler)
-        tap = GaiaClass(conn_handler, tapplus)
+        tap = GaiaClass(conn_handler, tapplus, show_server_messages=False)
         tap.login_gui()
         assert (mock_login_gui.call_count == 1)
         mock_login_gui.side_effect = HTTPError("Login error")
@@ -602,7 +642,7 @@ def test_login_gui(self, mock_login_gui, mock_login):
     def test_logout(self, mock_logout):
         conn_handler = DummyConnHandler()
         tapplus = TapPlus("http://test:1111/tap", connhandler=conn_handler)
-        tap = GaiaClass(conn_handler, tapplus)
+        tap = GaiaClass(conn_handler, tapplus, show_server_messages=False)
         tap.logout()
         assert (mock_logout.call_count == 2)
         mock_logout.side_effect = HTTPError("Login error")