Rename **kwargs, info to regenerate column file, parameter checking on collections

Author: snbianco

astroquery/mast/collections.py

@@ -6,6 +6,7 @@
 This module contains various methods for querying MAST collections such as catalogs.
 """
 
+import difflib
 import warnings
 import os
 import time
@@ -60,7 +61,7 @@ def _parse_result(self, response, *, verbose=False):
 
     @class_or_instance
     def query_region_async(self, coordinates, *, radius=0.2*u.deg, catalog="Hsc",
-                           version=None, pagesize=None, page=None, **kwargs):
+                           version=None, pagesize=None, page=None, **criteria):
         """
         Given a sky position and radius, returns a list of catalog entries.
         See column documentation for specific catalogs `here <https://mast.stsci.edu/api/v0/pages.html>`__.
@@ -88,7 +89,7 @@ def query_region_async(self, coordinates, *, radius=0.2*u.deg, catalog="Hsc",
             Default None.
             Can be used to override the default behavior of all results being returned to obtain a
             specific page of results.
-        **kwargs
+        **criteria
             Other catalog-specific keyword args.
             These can be found in the (service documentation)[https://mast.stsci.edu/api/v0/_services.html]
             for specific catalogs. For example one can specify the magtype for an HSC search.
@@ -109,10 +110,18 @@ def query_region_async(self, coordinates, *, radius=0.2*u.deg, catalog="Hsc",
                   'dec': coordinates.dec.deg,
                   'radius': radius.deg}
 
+        # valid criteria keywords
+        valid_criteria = []
+
         # Determine API connection and service name
         if catalog.lower() in self._service_api_connection.SERVICES:
             self._current_connection = self._service_api_connection
             service = catalog
+
+            # adding additional user specified parameters
+            for prop, value in criteria.items():
+                params[prop] = value
+
         else:
             self._current_connection = self._portal_api_connection
 
@@ -125,19 +134,20 @@ def query_region_async(self, coordinates, *, radius=0.2*u.deg, catalog="Hsc",
                         warnings.warn("Invalid HSC version number, defaulting to v3.", InputWarning)
                     service = "Mast.Hsc.Db.v3"
 
-                self.catalog_limit = kwargs.get('nr', 50000)
-
                 # Hsc specific parameters (can be overridden by user)
-                params['nr'] = 50000
-                params['ni'] = 1
-                params['magtype'] = 1
+                self.catalog_limit = criteria.pop('nr', 50000)
+                valid_criteria = ['nr', 'ni', 'magtype']
+                params['nr'] = self.catalog_limit
+                params['ni'] = criteria.pop('ni', 1)
+                params['magtype'] = criteria.pop('magtype', 1)
 
             elif catalog.lower() == "galex":
                 service = "Mast.Galex.Catalog"
-                self.catalog_limit = kwargs.get('maxrecords', 50000)
+                self.catalog_limit = criteria.get('maxrecords', 50000)
 
                 # galex specific parameters (can be overridden by user)
-                params['maxrecords'] = 50000
+                valid_criteria = ['maxrecords']
+                params['maxrecords'] = criteria.pop('maxrecords', 50000)
 
             elif catalog.lower() == "gaia":
                 if version == 1:
@@ -158,9 +168,16 @@ def query_region_async(self, coordinates, *, radius=0.2*u.deg, catalog="Hsc",
                 service = "Mast.Catalogs." + catalog + ".Cone"
                 self.catalog_limit = None
 
-        # adding additional user specified parameters
-        for prop, value in kwargs.items():
-            params[prop] = value
+            # additional user-specified parameters are not valid
+            if criteria:
+                key = next(iter(criteria))
+                closest_match = difflib.get_close_matches(key, valid_criteria, n=1)
+                error_msg = (
+                    f"Filter '{key}' does not exist for catalog {catalog}. Did you mean '{closest_match[0]}'?"
+                    if closest_match
+                    else f"Filter '{key}' does not exist for catalog {catalog}."
+                )
+                raise InvalidQueryError(error_msg)
 
         # Parameters will be passed as JSON objects only when accessing the PANSTARRS API
         use_json = catalog.lower() == 'panstarrs'
@@ -170,7 +187,7 @@ def query_region_async(self, coordinates, *, radius=0.2*u.deg, catalog="Hsc",
 
     @class_or_instance
     def query_object_async(self, objectname, *, radius=0.2*u.deg, catalog="Hsc",
-                           pagesize=None, page=None, version=None, **kwargs):
+                           pagesize=None, page=None, version=None, **criteria):
         """
         Given an object name, returns a list of catalog entries.
         See column documentation for specific catalogs `here <https://mast.stsci.edu/api/v0/pages.html>`__.
@@ -197,7 +214,7 @@ def query_object_async(self, objectname, *, radius=0.2*u.deg, catalog="Hsc",
             to obtain a specific page of results.
         version : int, optional
             Version number for catalogs that have versions. Default is highest version.
-        **kwargs
+        **criteria
             Catalog-specific keyword args.
             These can be found in the `service documentation <https://mast.stsci.edu/api/v0/_services.html>`__.
             for specific catalogs. For example one can specify the magtype for an HSC search.
@@ -215,7 +232,7 @@ def query_object_async(self, objectname, *, radius=0.2*u.deg, catalog="Hsc",
                                        version=version,
                                        pagesize=pagesize,
                                        page=page,
-                                       **kwargs)
+                                       **criteria)
 
     @class_or_instance
     def query_criteria_async(self, catalog, *, pagesize=None, page=None, **criteria):
@@ -295,26 +312,25 @@ def query_criteria_async(self, catalog, *, pagesize=None, page=None, **criteria)
                 if coordinates or objectname:
                     service += ".Position"
                 service += ".Rows"  # Using the rowstore version of the query for speed
-                filters = self._current_connection.build_filter_set("Mast.Catalogs.Tess.Cone",
-                                                                    service, **criteria)
+                column_config_name = "Mast.Catalogs.Tess.Cone"
                 params["columns"] = "*"
             elif catalog.lower() == "ctl":
                 service = "Mast.Catalogs.Filtered.Ctl"
                 if coordinates or objectname:
                     service += ".Position"
                 service += ".Rows"  # Using the rowstore version of the query for speed
-                filters = self._current_connection.build_filter_set("Mast.Catalogs.Tess.Cone",
-                                                                    service, **criteria)
+                column_config_name = "Mast.Catalogs.Tess.Cone"
                 params["columns"] = "*"
             elif catalog.lower() == "diskdetective":
                 service = "Mast.Catalogs.Filtered.DiskDetective"
                 if coordinates or objectname:
                     service += ".Position"
-                filters = self._current_connection.build_filter_set("Mast.Catalogs.Dd.Cone",
-                                                                    service, **criteria)
+                column_config_name = "Mast.Catalogs.Dd.Cone"
             else:
                 raise InvalidQueryError("Criteria query not available for {}".format(catalog))
 
+            filters = self._current_connection.build_filter_set(column_config_name, service, **criteria)
+
             if not filters:
                 raise InvalidQueryError("At least one non-positional criterion must be supplied.")
             params["filters"] = filters

astroquery/mast/missions.py

@@ -77,7 +77,7 @@ def _validate_criteria(self, **criteria):
 
         Parameters
         ----------
-        **kwargs
+        **criteria
             Keyword arguments representing criteria filters to apply.
 
         Raises
@@ -102,7 +102,7 @@ def _validate_criteria(self, **criteria):
                 raise InvalidQueryError(error_msg)
 
     @class_or_instance
-    def query_region_async(self, coordinates, *, radius=3*u.arcmin, limit=5000, offset=0, **kwargs):
+    def query_region_async(self, coordinates, *, radius=3*u.arcmin, limit=5000, offset=0, **criteria):
         """
         Given a sky position and radius, returns a list of matching dataset IDs.
 
@@ -122,12 +122,11 @@ def query_region_async(self, coordinates, *, radius=3*u.arcmin, limit=5000, offs
         offset : int
             Optional and default is 0
             the number of records you wish to skip before selecting records.
-        **kwargs
-            Other mission-specific keyword args.
-            Any invalid keys are ignored by the API.
-            All valid key names can be found using `~astroquery.mast.missions.MastMissionsClass.get_column_list`
+        **criteria
+            Other mission-specific criteria arguments.
+            All valid filters can be found using `~astroquery.mast.missions.MastMissionsClass.get_column_list`
             function.
-            For example one can specify the output columns(select_cols) or use other filters(conditions)
+            For example, one can specify the output columns(select_cols) or use other filters(conditions).
 
         Returns
         -------
@@ -137,7 +136,7 @@ def query_region_async(self, coordinates, *, radius=3*u.arcmin, limit=5000, offs
         self.limit = limit
 
         # Check that criteria arguments are valid
-        self._validate_criteria(**kwargs)
+        self._validate_criteria(**criteria)
 
         # Put coordinates and radius into consistent format
         coordinates = commons.parse_coordinates(coordinates)
@@ -154,7 +153,7 @@ def query_region_async(self, coordinates, *, radius=3*u.arcmin, limit=5000, offs
 
         params['conditions'] = []
         # adding additional user specified parameters
-        for prop, value in kwargs.items():
+        for prop, value in criteria.items():
             if prop not in self._search_option_fields:
                 params['conditions'].append({prop: value})
             else:
@@ -189,12 +188,11 @@ def query_criteria_async(self, *, coordinates=None, objectname=None, radius=3*u.
         select_cols: list
             names of columns that will be included in the astropy table
         **criteria
-            Criteria to apply. At least one non-positional criteria must be supplied.
+            Criteria to apply. At least one non-positional criterion must be supplied.
             Valid criteria are coordinates, objectname, radius (as in
             `~astroquery.mast.missions.MastMissionsClass.query_region` and
             `~astroquery.mast.missions.MastMissionsClass.query_object` functions),
             and all fields listed in the column documentation for the mission being queried.
-            Any invalid keys passed in criteria are ignored by the API.
             List of all valid fields that can be used to match results on criteria can be retrieved by calling
             `~astroquery.mast.missions.MastMissionsClass.get_column_list` function.
 
@@ -234,7 +232,7 @@ def query_criteria_async(self, *, coordinates=None, objectname=None, radius=3*u.
         return self._service_api_connection.service_request_async(self.service, params, use_json=True)
 
     @class_or_instance
-    def query_object_async(self, objectname, *, radius=3*u.arcmin, limit=5000, offset=0, **kwargs):
+    def query_object_async(self, objectname, *, radius=3*u.arcmin, limit=5000, offset=0, **criteria):
         """
         Given an object name, returns a list of matching rows.
 
@@ -253,11 +251,11 @@ def query_object_async(self, objectname, *, radius=3*u.arcmin, limit=5000, offse
         offset : int
             Optional and default is 0.
             the number of records you wish to skip before selecting records.
-        **kwargs
-            Mission-specific keyword args.
-            Any invalid keys are ignored by the API.
-            All valid keys can be found by calling `~astroquery.mast.missions.MastMissionsClass.get_column_list`
+        **criteria
+            Other mission-specific criteria arguments.
+            All valid filters can be found using `~astroquery.mast.missions.MastMissionsClass.get_column_list`
             function.
+            For example, one can specify the output columns(select_cols) or use other filters(conditions).
 
         Returns
         -------
@@ -266,7 +264,7 @@ def query_object_async(self, objectname, *, radius=3*u.arcmin, limit=5000, offse
 
         coordinates = utils.resolve_object(objectname)
 
-        return self.query_region_async(coordinates, radius=radius, limit=limit, offset=offset, **kwargs)
+        return self.query_region_async(coordinates, radius=radius, limit=limit, offset=offset, **criteria)
 
     @class_or_instance
     def get_column_list(self):
@@ -294,8 +292,8 @@ def get_column_list(self):
                 # Create Table with parsed data
                 col_table = Table(rows=rows, names=('name', 'data_type', 'description'))
                 self.columns[self.mission] = col_table
-            except Exception:
-                raise Exception(f'Error occurred while trying to get column list for mission {self.mission}')
+            except Exception as ex:
+                raise RuntimeError(f'Error occurred while trying to get column list for mission {self.mission}: {ex}')
 
         return self.columns[self.mission]
 

astroquery/mast/tests/data/README.rst

@@ -0,0 +1,17 @@
+===============
+MAST Test Data
+===============
+
+This directory contains sample data that is used to mock functions in `~astroquery.mast.tests.test_mast.py`.
+
+To generate `~astroquery.mast.tests.data.mission_columns.json`, use the following:
+
+.. code-block:: python
+
+    >>> import json
+    >>> from astroquery.mast import utils
+
+    >>> params = {'mission': 'hst'}
+    >>> resp = utils._simple_request(f'https://mast.stsci.edu/search/util/api/v0.1/column_list', {'mission': 'hst'})
+    >>> with open('mission_columns.json', 'w') as file:
+    >>>     json.dump(resp.json(), file, indent=4)