Merge pull request #2077 from esdc-esac-esa-int/gaia-astroquery-1.0

Gaia astroquery 1.0

Author: bsipocz

CHANGES.rst

@@ -30,6 +30,10 @@ gaia
   ``astroquery.gaia.Gaia`` no longer ignore their ``columns`` argument when
   ``radius`` is specified. [#2249]
 
+- Enhanced methods 'launch_job' and 'launch_job_async' to avoid issues with
+  the name provided by the user for the output file when the results are
+  returned by the TAP in compressed format. [#2077]
+
 mast
 ^^^^
 
@@ -48,6 +52,7 @@ sdss
 - Fix ``query_crossid`` to be able to query larger list of coordinates. [#2305]
 
 
+
 Infrastructure, Utility and Other Changes and Additions
 -------------------------------------------------------
 
@@ -57,10 +62,17 @@ Infrastructure, Utility and Other Changes and Additions
 - Callback hooks are deleted before caching.  Potentially all cached queries
   prior to this PR will be rendered invalid.  [#2295]
 
+utils.tap
+^^^^^^^^^
+
 - The modules that make use of the ``astroquery.utils.tap.model.job.Job`` class
   (e.g. Gaia) no longer print messages about where the results of async queries
   were written if the ``verbose`` setting is ``False``. [#2299]
 
+- New method, ``rename_table``, which allows the user to rename table and
+  column names. [#2077]
+
+
 
 0.4.5 (2021-12-24)
 ==================

astroquery/gaia/core.py

@@ -24,7 +24,7 @@
 import mimetypes
 import time
 
-from urllib.parse import urlencode
+from urllib.parse import urlencode
 
 from astroquery.utils.tap.xmlparser import utils
 from astroquery.utils.tap import taputils
@@ -87,7 +87,7 @@ def __init__(self, ishttps,
         self.__connPort = port
         self.__connPortSsl = sslport
         if server_context is not None:
-            if(server_context.startswith("/")):
+            if server_context.startswith("/"):
                 self.__serverContext = server_context
             else:
                 self.__serverContext = f"/{server_context}"
@@ -106,8 +106,8 @@ def __init__(self, ishttps,
             self.__connectionHandler = connhandler
 
     def __create_context(self, context):
-        if (context is not None and context != ""):
-            if(str(context).startswith("/")):
+        if context is not None and context != "":
+            if str(context).startswith("/"):
                 return f"{self.__serverContext}{context}"
             else:
                 return f"{self.__serverContext}/{context}"
@@ -391,6 +391,29 @@ def execute_table_edit(self, data,
         context = self.__get_table_edit_context()
         return self.__execute_post(context, data, content_type, verbose)
 
+    def execute_table_tool(self, data,
+                           content_type=CONTENT_TYPE_POST_DEFAULT,
+                           verbose=False):
+        """Executes a POST upload request
+        The connection is done through HTTP or HTTPS depending on the login
+        status (logged in -> HTTPS)
+
+        Parameters
+        ----------
+        data : str, mandatory
+            POST data
+        content_type: str, optional, default: application/x-www-form-urlencoded
+            HTTP(s) content-type header value
+        verbose : bool, optional, default 'False'
+            flag to display information about the process
+
+        Returns
+        -------
+        An HTTP(s) response object
+        """
+        context = self.__get_table_edit_context()
+        return self.__execute_post(context, data, content_type, verbose)
+
     def __execute_post(self, context, data,
                        content_type=CONTENT_TYPE_POST_DEFAULT,
                        verbose=False):

astroquery/gaia/tests/data/job_2.vot.gz

@@ -170,3 +170,9 @@ def execute_table_edit(self, data,
                            verbose=False):
         return self.__execute_post(subcontext="tableEdit", data=data,
                                    content_type=content_type, verbose=verbose)
+
+    def execute_table_tool(self, data,
+                           content_type="application/x-www-form-urlencoded",
+                           verbose=False):
+        return self.__execute_post(subcontext="TableTool", data=data,
+                                   content_type=content_type, verbose=verbose)

astroquery/gaia/tests/test_gaiatap.py

@@ -11,8 +11,7 @@
 European Space Agency (ESA)
 
 Created on 30 jun. 2016
-
-
+Modified on 1 jun. 2021 by mhsarmiento
 """
 from astroquery.utils.tap import taputils
 from astroquery.utils.tap.conn.tapconn import TapConn
@@ -32,7 +31,6 @@
 from astropy.table.table import Table
 import tempfile
 
-
 __all__ = ['Tap', 'TapPlus']
 
 VERSION = "20200428.1"
@@ -246,6 +244,8 @@ def launch_job(self, query, name=None, output_file=None,
         ----------
         query : str, mandatory
             query to be executed
+        name : str, optional, default None
+            custom name defined by the user for the job that is going to be created
         output_file : str, optional, default None
             file name where the results are saved if dumpToFile is True.
             If this parameter is not provided, the jobid is used instead
@@ -358,6 +358,8 @@ def launch_job_async(self, query, name=None, output_file=None,
         ----------
         query : str, mandatory
             query to be executed
+        name : str, optional, default None
+            custom name defined by the user for the job that is going to be created
         output_file : str, optional, default None
             file name where the results are saved if dumpToFile is True.
             If this parameter is not provided, the jobid is used instead
@@ -571,7 +573,7 @@ def __launchJobMultipart(self, query, uploadResource, uploadTableName,
             "FORMAT": str(outputFormat),
             "tapclient": str(TAP_CLIENT_ID),
             "QUERY": str(query),
-            "UPLOAD": ""+str(uploadValue)}
+            "UPLOAD": "" + str(uploadValue)}
         if autorun is True:
             args['PHASE'] = 'RUN'
         if name is not None:
@@ -657,7 +659,7 @@ def __parseUrl(self, url, verbose=False):
         if urlInfoPos < 0:
             raise ValueError("Invalid URL format")
 
-        urlInfo = url[(urlInfoPos+3):]
+        urlInfo = url[(urlInfoPos + 3):]
 
         items = urlInfo.split("/")
 
@@ -672,7 +674,7 @@ def __parseUrl(self, url, verbose=False):
         if portPos > 0:
             # port found
             host = hostPort[0:portPos]
-            port = int(hostPort[portPos+1:])
+            port = int(hostPort[portPos + 1:])
         else:
             # no port found
             host = hostPort
@@ -693,10 +695,10 @@ def __parseUrl(self, url, verbose=False):
             tapContext = f"/{items[2]}"
         else:
             data = []
-            for i in range(1, itemsSize-1):
+            for i in range(1, itemsSize - 1):
                 data.append(f"/{items[i]}")
             serverContext = utils.util_create_string_from_buffer(data)
-            tapContext = f"/{items[itemsSize-1]}"
+            tapContext = f"/{items[itemsSize - 1]}"
         if verbose:
             print(f"protocol: '{protocol}'")
             print(f"host: '{host}'")
@@ -713,6 +715,7 @@ class TapPlus(Tap):
     """TAP plus class
     Provides TAP and TAP+ capabilities
     """
+
     def __init__(self, url=None,
                  host=None,
                  server_context=None,
@@ -1245,7 +1248,7 @@ def get_datalinks(self, ids, verbose=False):
             print("Retrieving datalink.")
         if ids is None:
             raise ValueError("Missing mandatory argument 'ids'")
-        if isinstance(ids, str):
+        if isinstance(ids, str):
             ids_arg = f"ID={ids}"
         else:
             if isinstance(ids, int):
@@ -1368,7 +1371,7 @@ def login(self, user=None, password=None, credentials_file=None,
                 user = ins.readline().strip()
                 password = ins.readline().strip()
         if user is None:
-            user = input("User: ")
+            user = input("User: ")
             if user is None:
                 print("Invalid user name")
                 return
@@ -1532,7 +1535,7 @@ def __uploadTableMultipart(self, resource, table_name=None,
                     chunk = f.read()
                 files = [['FILE', os.path.basename(resource), chunk]]
                 contentType, body = connHandler.encode_multipart(args, files)
-            else:    # upload from URL
+            else:  # upload from URL
                 args = {
                     "TASKID": str(-1),
                     "TABLE_NAME": str(table_name),
@@ -1628,14 +1631,14 @@ def delete_user_table(self, table_name=None, force_removal=False,
             raise ValueError("Table name cannot be null")
         if force_removal is True:
             args = {
-                    "TABLE_NAME": str(table_name),
-                    "DELETE": "TRUE",
-                    "FORCE_REMOVAL": "TRUE"}
+                "TABLE_NAME": str(table_name),
+                "DELETE": "TRUE",
+                "FORCE_REMOVAL": "TRUE"}
         else:
             args = {
-                    "TABLE_NAME": str(table_name),
-                    "DELETE": "TRUE",
-                    "FORCE_REMOVAL": "FALSE"}
+                "TABLE_NAME": str(table_name),
+                "DELETE": "TRUE",
+                "FORCE_REMOVAL": "FALSE"}
         connHandler = self.__getconnhandler()
         data = connHandler.url_encode(args)
         response = connHandler.execute_upload(data, verbose=verbose)
@@ -1648,15 +1651,90 @@ def delete_user_table(self, table_name=None, force_removal=False,
         msg = f"Table '{table_name}' deleted."
         print(msg)
 
+    def rename_table(self, table_name=None, new_table_name=None, new_column_names_dict=None,
+                     verbose=False):
+        """ This method allows you to update the column names of a user table.
+
+        Parameters
+        ----------
+        table_name: str, required
+            old name of the user's table
+        new_table_name: str, required
+            new name of the user's table
+        new_column_names_dict: dict str:str
+            dict with pairs "old_column1_name:new_column1_name"
+        verbose : bool, optional, default 'False'
+            flag to display information about the process
+
+        """
+        if new_column_names_dict is None:
+            new_column_names_dict = {}
+        args = {}
+
+        if table_name is None:
+            raise ValueError(
+                "Argument 'table_name' is mandatory. "
+                "Please introduce the name of the table that is going to be updated")
+        if (new_table_name is None) and (new_column_names_dict is None):
+            raise ValueError("Please introduce as minimum a new name for the table or a new name for a column with "
+                             "format old_column1_name:new_column1_name, ... ,old_columnN_name:new_columnN_name")
+        if new_table_name is not None:
+            if new_column_names_dict is None:
+                # case 1: We only need to rename the table
+                args = self.get_args_4_rename_table(table_name, new_table_name)
+            else:
+                # case 2: We need to rename both, table and column names
+                args = self.get_args_4_rename_table(table_name, new_table_name, new_column_names_dict)
+
+        if new_table_name is None:
+            if new_column_names_dict:
+                # case 3: We only need to rename the columns but same table name
+                args = self.get_args_4_rename_table(table_name, table_name, new_column_names_dict)
+
+        connHandler = self.__getconnhandler()
+        data = connHandler.url_encode(args)
+        response = connHandler.execute_table_tool(data, verbose=verbose)
+
+        if verbose:
+            print(response.status, response.reason)
+            print(response.getheaders())
+        connHandler.check_launch_response_status(response,
+                                                 verbose,
+                                                 200)
+        if verbose:
+            msg = f"Table '{table_name}' updated."
+            print(msg)
+
+    def get_args_4_rename_table(self, table_name, new_table_name, new_column_names_dict):
+
+        args = {}
+
+        if not new_column_names_dict:
+            args = {
+                "action": "rename",
+                "new_table_name": new_table_name,
+                "table_name": table_name
+            }
+        else:
+            new_column_names = ','.join(f'{key}:{value}' for key, value in new_column_names_dict.items())
+
+            args = {
+                "action": "rename",
+                "new_column_names": new_column_names,
+                "new_table_name": new_table_name,
+                "table_name": table_name
+            }
+        return args
+
     def update_user_table(self, table_name=None, list_of_changes=[],
                           verbose=False):
         """Updates a user table
 
         Parameters
         ----------
-        table_name : str, required
+        table_name : str
             table to be updated
-        list_of_changes : list, required
+        list_of_changes : list
             list of lists, each one of them containing sets of
             [column_name, field_name, value].
             column_name is the name of the column to be updated
@@ -1738,11 +1816,11 @@ def update_user_table(self, table_name=None, list_of_changes=[],
     def get_table_update_arguments(table_name, columns, list_of_changes):
         num_cols = len(columns)
         args = {
-                "ACTION": "edit",
-                "NUMTABLES": str(1),
-                "TABLE0_NUMCOLS": str(num_cols),
-                "TABLE0": str(table_name),
-                }
+            "ACTION": "edit",
+            "NUMTABLES": str(1),
+            "TABLE0_NUMCOLS": str(num_cols),
+            "TABLE0": str(table_name),
+        }
         index = 0
         for column in columns:
             found_in_changes = False
@@ -1872,11 +1950,11 @@ def set_ra_dec_columns(self, table_name=None,
 
         Parameters
         ----------
-        table_name : str, required
+        table_name : str
             table to be set
-        ra_column_name : str, required
+        ra_column_name : str
             ra column to be set
-        dec_column_name : str, required
+        dec_column_name : str
             dec column to be set
         verbose : bool, optional, default 'False'
             flag to display information about the process
@@ -1890,11 +1968,11 @@ def set_ra_dec_columns(self, table_name=None,
             raise ValueError("Dec column name cannot be null")
 
         args = {
-                "ACTION": "radec",
-                "TABLE_NAME": str(table_name),
-                "RA": str(ra_column_name),
-                "DEC": str(dec_column_name),
-                }
+            "ACTION": "radec",
+            "TABLE_NAME": str(table_name),
+            "RA": str(ra_column_name),
+            "DEC": str(dec_column_name),
+        }
         connHandler = self.__getconnhandler()
         data = connHandler.url_encode(args)
         response = connHandler.execute_table_edit(data, verbose=verbose)
@@ -1933,7 +2011,7 @@ def login(self, user=None, password=None, credentials_file=None,
                 user = ins.readline().strip()
                 password = ins.readline().strip()
         if user is None:
-            user = input("User: ")
+            user = input("User: ")
             if user is None:
                 log.info("Invalid user name")
                 return