Revert "updates to missions_mast search API astroquery interface"

This reverts commit e0a6a4d.

Author: syed-gilani

astroquery/mast/missions.py

@@ -7,15 +7,13 @@
 """
 
 import requests
-import warnings
 
-from astropy.table import Table
 import astropy.units as u
 import astropy.coordinates as coord
 
 from astroquery.utils import commons, async_to_sync
 from astroquery.utils.class_or_instance import class_or_instance
-from astroquery.exceptions import InvalidQueryError, MaxResultsWarning
+from astroquery.exceptions import InvalidQueryError
 
 from astroquery.mast import utils
 from astroquery.mast.core import MastQueryWithLogin
@@ -29,7 +27,8 @@
 class MastMissionsClass(MastQueryWithLogin):
     """
     MastMissions search class.
-    Class that allows direct programatic access to retrieve metadata via the MAST search API for a given mission.
+
+    Class that allows direct programatic access to the MAST search API for a given mission.
     """
 
     def __init__(self, *, mission='hst', service='search'):
@@ -39,12 +38,11 @@ def __init__(self, *, mission='hst', service='search'):
                                       'skip_count', 'user_fields']
         self.service = service
         self.mission = mission
-        self.limit = 5000
 
         service_dict = {self.service: {'path': self.service, 'args': {}}}
         self._service_api_connection.set_service_params(service_dict, f"{self.service}/{self.mission}")
 
-    def _parse_result(self, response, *, verbose=False):  # Used by the async_to_sync decorator functionality
+    def _parse_result(self, response, verbose=False):  # Used by the async_to_sync decorator functionality
         """
         Parse the results of a `~requests.Response` objects and return an `~astropy.table.Table` of results.
 
@@ -55,22 +53,17 @@ def _parse_result(self, response, *, verbose=False):  # Used by the async_to_syn
         verbose : bool
             (presently does nothing - there is no output with verbose set to
             True or False)
-            Default False. Setting to True provides more extensive output.
+            Default False.  Setting to True provides more extensive output.
 
         Returns
         -------
         response : `~astropy.table.Table`
         """
 
-        results = self._service_api_connection._parse_result(response, verbose, data_key='results')
-        if len(results) >= self.limit:
-            warnings.warn("Maximum results returned, may not include all sources within radius.",
-                          MaxResultsWarning)
-
-        return results
+        return self._service_api_connection._parse_result(response, verbose, data_key='results')
 
     @class_or_instance
-    def query_region_async(self, coordinates, *, radius=3*u.arcmin, limit=5000, offset=0, **kwargs):
+    def query_region_async(self, coordinates, radius=3*u.arcmin, **kwargs):
         """
         Given a sky position and radius, returns a list of matching dataset IDs.
 
@@ -84,26 +77,17 @@ def query_region_async(self, coordinates, *, radius=3*u.arcmin, limit=5000, offs
             The string must be parsable by `~astropy.coordinates.Angle`. The
             appropriate `~astropy.units.Quantity` object from
             `~astropy.units` may also be used. Defaults to 3 arcminutes.
-        limit : int
-            Optional and default is 5000.
-            the maximun number of dataset IDs in the results.
-        offset : int
-            Optional and default is 0
-            the number of records you wish to skip before selecting records.
         **kwargs
             Other mission-specific keyword args.
-            Any invalid keys are ignored by the API.
-            All valid key names can be found using `~astroquery.mast.missions.MastMissionsClass.get_column_list`
-            function.
+            These can be found at the following link
+            https://mast.stsci.edu/search/docs/#/Hubble%20Search/post_search_hst_api_v0_1_search_post
             For example one can specify the output columns(select_cols) or use other filters(conditions)
 
         Returns
         -------
         response : list of `~requests.Response`
         """
 
-        self.limit = limit
-
         # Put coordinates and radius into consistant format
         coordinates = commons.parse_coordinates(coordinates)
 
@@ -113,9 +97,7 @@ def query_region_async(self, coordinates, *, radius=3*u.arcmin, limit=5000, offs
         # basic params
         params = {'target': [f"{coordinates.ra.deg} {coordinates.dec.deg}"],
                   'radius': radius.arcmin,
-                  'radius_units': 'arcminutes',
-                  'limit': limit,
-                  'offset': offset}
+                  'radius_units': 'arcminutes'}
 
         params['conditions'] = []
         # adding additional user specified parameters
@@ -128,47 +110,29 @@ def query_region_async(self, coordinates, *, radius=3*u.arcmin, limit=5000, offs
         return self._service_api_connection.service_request_async(self.service, params, use_json=True)
 
     @class_or_instance
-    def query_criteria_async(self, *, coordinates=None, objectname=None, radius=3*u.arcmin,
-                             limit=5000, offset=0, select_cols=[], **criteria):
+    def query_criteria_async(self, **criteria):
         """
         Given a set of search criteria, returns a list of mission metadata.
 
         Parameters
         ----------
-        coordinates : str or `~astropy.coordinates` object
-            The target around which to search. It may be specified as a
-            string or as the appropriate `~astropy.coordinates` object.
-        objectname : str
-            The name of the target around which to search.
-        radius : str or `~astropy.units.Quantity` object, optional
-            Default 3 degrees.
-            The string must be parsable by `~astropy.coordinates.Angle`. The
-            appropriate `~astropy.units.Quantity` object from
-            `~astropy.units` may also be used. Defaults to 3 arcminutes.
-        limit : int
-            Optional and default is 5000.
-            the maximun number of dataset IDs in the results.
-        offset : int
-            Optional and default is 0.
-            the number of records you wish to skip before selecting records.
-        select_cols: list
-            names of columns that will be included in the astropy table
         **criteria
             Criteria to apply. At least one non-positional criteria must be supplied.
-            Valid criteria are coordinates, objectname, radius (as in
-            `~astroquery.mast.missions.MastMissionsClass.query_region` and
-            `~astroquery.mast.missions.MastMissionsClass.query_object` functions),
+            Valid criteria are coordinates, objectname, radius (as in `query_region` and `query_object`),
             and all fields listed in the column documentation for the mission being queried.
-            Any invalid keys passed in criteria are ignored by the API.
-            List of all valid fields that can be used to match results on criteria can be retrieved by calling
-            `~astroquery.mast.missions.MastMissionsClass.get_column_list` function.
+            Fields that can be used to match results on criteria. See the TAP schema link below for all field names.
+            https://vao.stsci.edu/missionmast/tapservice.aspx/tables
+            some common fields for criteria are sci_pep_id, sci_spec_1234 and sci_actual_duration.
 
         Returns
         -------
         response : list of `~requests.Response`
         """
 
-        self.limit = limit
+        # Seperating any position info from the rest of the filters
+        coordinates = criteria.pop('coordinates', None)
+        objectname = criteria.pop('objectname', None)
+        radius = criteria.pop('radius', 0.2*u.deg)
 
         if objectname or coordinates:
             coordinates = utils.parse_input_location(coordinates, objectname)
@@ -177,7 +141,7 @@ def query_criteria_async(self, *, coordinates=None, objectname=None, radius=3*u.
         radius = coord.Angle(radius, u.arcmin)
 
         # build query
-        params = {"limit": self.limit, "offset": offset, 'select_cols': select_cols}
+        params = {}
         if coordinates:
             params["target"] = [f"{coordinates.ra.deg} {coordinates.dec.deg}"]
             params["radius"] = radius.arcmin
@@ -196,7 +160,7 @@ def query_criteria_async(self, *, coordinates=None, objectname=None, radius=3*u.
         return self._service_api_connection.service_request_async(self.service, params, use_json=True)
 
     @class_or_instance
-    def query_object_async(self, objectname, *, radius=3*u.arcmin, limit=5000, offset=0, **kwargs):
+    def query_object_async(self, objectname, radius=3*u.arcmin, **kwargs):
         """
         Given an object name, returns a list of matching rows.
 
@@ -209,17 +173,10 @@ def query_object_async(self, objectname, *, radius=3*u.arcmin, limit=5000, offse
             The string must be parsable by `~astropy.coordinates.Angle`.
             The appropriate `~astropy.units.Quantity` object from
             `~astropy.units` may also be used. Defaults to 3 arcminutes.
-        limit : int
-            Optional and default is 5000.
-            the maximun number of dataset IDs in the results.
-        offset : int
-            Optional and default is 0.
-            the number of records you wish to skip before selecting records.
         **kwargs
             Mission-specific keyword args.
-            Any invalid keys are ignored by the API.
-            All valid keys can be found by calling `~astroquery.mast.missions.MastMissionsClass.get_column_list`
-            function.
+            These can be found in the `service documentation <https://mast.stsci.edu/api/v0/_services.html>`__.
+            for specific catalogs. For example one can specify the magtype for an HSC search.
 
         Returns
         -------
@@ -228,7 +185,7 @@ def query_object_async(self, objectname, *, radius=3*u.arcmin, limit=5000, offse
 
         coordinates = utils.resolve_object(objectname)
 
-        return self.query_region_async(coordinates, radius=radius, limit=limit, offset=offset, **kwargs)
+        return self.query_region_async(coordinates, radius, **kwargs)
 
     @class_or_instance
     def get_column_list(self):
@@ -237,23 +194,20 @@ def get_column_list(self):
 
         Returns
         -------
-        response : `~astropy.table.Table` that contains columns names, types  and their descriptions
+        json data that contains columns names and their descriptions
         """
 
         url = f"{conf.server}/search/util/api/v0.1/column_list?mission={self.mission}"
 
         try:
             results = requests.get(url)
             results = results.json()
-            rows = []
             for result in results:
                 result.pop('field_name')
                 result.pop('queryable')
                 result.pop('indexed')
                 result.pop('default_output')
-                rows.append((result['column_name'], result['qual_type'], result['description']))
-            data_table = Table(rows=rows, names=('name', 'data_type', 'description'))
-            return data_table
+            return results
         except Exception:
             raise Exception(f"Error occured while trying to get column list for mission {self.mission}")
 

astroquery/mast/services.py

@@ -31,7 +31,7 @@ def _json_to_table(json_obj, data_key='data'):
 
     Parameters
     ----------
-    json_obj : data array or list of dictionaries
+    json_obj : dict
         A MAST microservice response JSON object (python dictionary)
     data_key : str
         string that contains the key name in json_obj that stores the data rows
@@ -50,7 +50,6 @@ def _json_to_table(json_obj, data_key='data'):
 
     # for each item in info, store the type and column name
     # for each item in info, type has to be converted from DB data types (SQL server in most cases)
-    # from missions_mast search service such as varchar, integer, float, boolean etc
     # to corresponding numpy type
     for idx, col, col_type, ignore_value in \
             [(idx, x['name'], x[type_key].lower(), None) for idx, x in enumerate(json_obj['info'])]:
@@ -79,7 +78,6 @@ def _json_to_table(json_obj, data_key='data'):
             # Step through data array of values
             col_data = np.array([x[idx] for x in json_obj[data_key]], dtype=object)
         except KeyError:
-            # it's not a data array, fall back to using column name as it is array of dictionaries
             col_data = np.array([x[col] for x in json_obj[data_key]], dtype=object)
         if ignore_value is not None:
             col_data[np.where(np.equal(col_data, None))] = ignore_value

astroquery/mast/tests/data/mission_incorrect_results.json

@@ -131,5 +131,5 @@
       }
     ]
   },
-  "totalResults": 3
+  "totalResults": 3,
 }

astroquery/mast/tests/test_mast.py

@@ -1,6 +1,5 @@
 # Licensed under a 3-clause BSD style license - see LICENSE.rst
 
-import json
 import os
 import re
 from shutil import copyfile
@@ -13,7 +12,6 @@
 
 import astropy.units as u
 
-from astroquery.mast.services import _json_to_table
 from astroquery.utils.mocks import MockResponse
 from astroquery.exceptions import InvalidQueryError, InputWarning
 
@@ -260,7 +258,6 @@ def test_missions_query_criteria_async_with_missing_results(patch_post):
                                                                        obs_type,
                                                                        aec,
                                                                        aperture])
-        table = _json_to_table(json.loads(responses), 'results')
 
 
 ###################