Merge pull request #2321 from syed-gilani/missions_mast

Missions mast

Author: bsipocz

astroquery/mast/missions.py

@@ -7,13 +7,15 @@
 """
 
 import requests
+import warnings
 
+from astropy.table import Table
 import astropy.units as u
 import astropy.coordinates as coord
 
 from astroquery.utils import commons, async_to_sync
 from astroquery.utils.class_or_instance import class_or_instance
-from astroquery.exceptions import InvalidQueryError
+from astroquery.exceptions import InvalidQueryError, MaxResultsWarning
 
 from astroquery.mast import utils
 from astroquery.mast.core import MastQueryWithLogin
@@ -27,8 +29,7 @@
 class MastMissionsClass(MastQueryWithLogin):
     """
     MastMissions search class.
-
-    Class that allows direct programatic access to the MAST search API for a given mission.
+    Class that allows direct programatic access to retrieve metadata via the MAST search API for a given mission.
     """
 
     def __init__(self, *, mission='hst', service='search'):
@@ -38,11 +39,12 @@ def __init__(self, *, mission='hst', service='search'):
                                       'skip_count', 'user_fields']
         self.service = service
         self.mission = mission
+        self.limit = 5000
 
         service_dict = {self.service: {'path': self.service, 'args': {}}}
         self._service_api_connection.set_service_params(service_dict, f"{self.service}/{self.mission}")
 
-    def _parse_result(self, response, verbose=False):  # Used by the async_to_sync decorator functionality
+    def _parse_result(self, response, *, verbose=False):  # Used by the async_to_sync decorator functionality
         """
         Parse the results of a `~requests.Response` objects and return an `~astropy.table.Table` of results.
 
@@ -53,17 +55,22 @@ def _parse_result(self, response, verbose=False):  # Used by the async_to_sync d
         verbose : bool
             (presently does nothing - there is no output with verbose set to
             True or False)
-            Default False.  Setting to True provides more extensive output.
+            Default False. Setting to True provides more extensive output.
 
         Returns
         -------
         response : `~astropy.table.Table`
         """
 
-        return self._service_api_connection._parse_result(response, verbose, data_key='results')
+        results = self._service_api_connection._parse_result(response, verbose, data_key='results')
+        if len(results) >= self.limit:
+            warnings.warn("Maximum results returned, may not include all sources within radius.",
+                          MaxResultsWarning)
+
+        return results
 
     @class_or_instance
-    def query_region_async(self, coordinates, radius=3*u.arcmin, **kwargs):
+    def query_region_async(self, coordinates, *, radius=3*u.arcmin, limit=5000, offset=0, **kwargs):
         """
         Given a sky position and radius, returns a list of matching dataset IDs.
 
@@ -77,17 +84,26 @@ def query_region_async(self, coordinates, radius=3*u.arcmin, **kwargs):
             The string must be parsable by `~astropy.coordinates.Angle`. The
             appropriate `~astropy.units.Quantity` object from
             `~astropy.units` may also be used. Defaults to 3 arcminutes.
+        limit : int
+            Optional and default is 5000.
+            the maximun number of dataset IDs in the results.
+        offset : int
+            Optional and default is 0
+            the number of records you wish to skip before selecting records.
         **kwargs
             Other mission-specific keyword args.
-            These can be found at the following link
-            https://mast.stsci.edu/search/docs/#/Hubble%20Search/post_search_hst_api_v0_1_search_post
+            Any invalid keys are ignored by the API.
+            All valid key names can be found using `~astroquery.mast.missions.MastMissionsClass.get_column_list`
+            function.
             For example one can specify the output columns(select_cols) or use other filters(conditions)
 
         Returns
         -------
         response : list of `~requests.Response`
         """
 
+        self.limit = limit
+
         # Put coordinates and radius into consistant format
         coordinates = commons.parse_coordinates(coordinates)
 
@@ -97,7 +113,9 @@ def query_region_async(self, coordinates, radius=3*u.arcmin, **kwargs):
         # basic params
         params = {'target': [f"{coordinates.ra.deg} {coordinates.dec.deg}"],
                   'radius': radius.arcmin,
-                  'radius_units': 'arcminutes'}
+                  'radius_units': 'arcminutes',
+                  'limit': limit,
+                  'offset': offset}
 
         params['conditions'] = []
         # adding additional user specified parameters
@@ -110,29 +128,47 @@ def query_region_async(self, coordinates, radius=3*u.arcmin, **kwargs):
         return self._service_api_connection.service_request_async(self.service, params, use_json=True)
 
     @class_or_instance
-    def query_criteria_async(self, **criteria):
+    def query_criteria_async(self, *, coordinates=None, objectname=None, radius=3*u.arcmin,
+                             limit=5000, offset=0, select_cols=[], **criteria):
         """
         Given a set of search criteria, returns a list of mission metadata.
 
         Parameters
         ----------
+        coordinates : str or `~astropy.coordinates` object
+            The target around which to search. It may be specified as a
+            string or as the appropriate `~astropy.coordinates` object.
+        objectname : str
+            The name of the target around which to search.
+        radius : str or `~astropy.units.Quantity` object, optional
+            Default 3 degrees.
+            The string must be parsable by `~astropy.coordinates.Angle`. The
+            appropriate `~astropy.units.Quantity` object from
+            `~astropy.units` may also be used. Defaults to 3 arcminutes.
+        limit : int
+            Optional and default is 5000.
+            the maximun number of dataset IDs in the results.
+        offset : int
+            Optional and default is 0.
+            the number of records you wish to skip before selecting records.
+        select_cols: list
+            names of columns that will be included in the astropy table
         **criteria
             Criteria to apply. At least one non-positional criteria must be supplied.
-            Valid criteria are coordinates, objectname, radius (as in `query_region` and `query_object`),
+            Valid criteria are coordinates, objectname, radius (as in
+            `~astroquery.mast.missions.MastMissionsClass.query_region` and
+            `~astroquery.mast.missions.MastMissionsClass.query_object` functions),
             and all fields listed in the column documentation for the mission being queried.
-            Fields that can be used to match results on criteria. See the TAP schema link below for all field names.
-            https://vao.stsci.edu/missionmast/tapservice.aspx/tables
-            some common fields for criteria are sci_pep_id, sci_spec_1234 and sci_actual_duration.
+            Any invalid keys passed in criteria are ignored by the API.
+            List of all valid fields that can be used to match results on criteria can be retrieved by calling
+            `~astroquery.mast.missions.MastMissionsClass.get_column_list` function.
 
         Returns
         -------
         response : list of `~requests.Response`
         """
 
-        # Seperating any position info from the rest of the filters
-        coordinates = criteria.pop('coordinates', None)
-        objectname = criteria.pop('objectname', None)
-        radius = criteria.pop('radius', 0.2*u.deg)
+        self.limit = limit
 
         if objectname or coordinates:
             coordinates = utils.parse_input_location(coordinates, objectname)
@@ -141,7 +177,7 @@ def query_criteria_async(self, **criteria):
         radius = coord.Angle(radius, u.arcmin)
 
         # build query
-        params = {}
+        params = {"limit": self.limit, "offset": offset, 'select_cols': select_cols}
         if coordinates:
             params["target"] = [f"{coordinates.ra.deg} {coordinates.dec.deg}"]
             params["radius"] = radius.arcmin
@@ -160,7 +196,7 @@ def query_criteria_async(self, **criteria):
         return self._service_api_connection.service_request_async(self.service, params, use_json=True)
 
     @class_or_instance
-    def query_object_async(self, objectname, radius=3*u.arcmin, **kwargs):
+    def query_object_async(self, objectname, *, radius=3*u.arcmin, limit=5000, offset=0, **kwargs):
         """
         Given an object name, returns a list of matching rows.
 
@@ -173,10 +209,17 @@ def query_object_async(self, objectname, radius=3*u.arcmin, **kwargs):
             The string must be parsable by `~astropy.coordinates.Angle`.
             The appropriate `~astropy.units.Quantity` object from
             `~astropy.units` may also be used. Defaults to 3 arcminutes.
+        limit : int
+            Optional and default is 5000.
+            the maximun number of dataset IDs in the results.
+        offset : int
+            Optional and default is 0.
+            the number of records you wish to skip before selecting records.
         **kwargs
             Mission-specific keyword args.
-            These can be found in the `service documentation <https://mast.stsci.edu/api/v0/_services.html>`__.
-            for specific catalogs. For example one can specify the magtype for an HSC search.
+            Any invalid keys are ignored by the API.
+            All valid keys can be found by calling `~astroquery.mast.missions.MastMissionsClass.get_column_list`
+            function.
 
         Returns
         -------
@@ -185,7 +228,7 @@ def query_object_async(self, objectname, radius=3*u.arcmin, **kwargs):
 
         coordinates = utils.resolve_object(objectname)
 
-        return self.query_region_async(coordinates, radius, **kwargs)
+        return self.query_region_async(coordinates, radius=radius, limit=limit, offset=offset, **kwargs)
 
     @class_or_instance
     def get_column_list(self):
@@ -194,20 +237,23 @@ def get_column_list(self):
 
         Returns
         -------
-        json data that contains columns names and their descriptions
+        response : `~astropy.table.Table` that contains columns names, types  and their descriptions
         """
 
         url = f"{conf.server}/search/util/api/v0.1/column_list?mission={self.mission}"
 
         try:
             results = requests.get(url)
             results = results.json()
+            rows = []
             for result in results:
                 result.pop('field_name')
                 result.pop('queryable')
                 result.pop('indexed')
                 result.pop('default_output')
-            return results
+                rows.append((result['column_name'], result['qual_type'], result['description']))
+            data_table = Table(rows=rows, names=('name', 'data_type', 'description'))
+            return data_table
         except Exception:
             raise Exception(f"Error occured while trying to get column list for mission {self.mission}")
 

astroquery/mast/services.py

@@ -31,7 +31,7 @@ def _json_to_table(json_obj, data_key='data'):
 
     Parameters
     ----------
-    json_obj : dict
+    json_obj : data array or list of dictionaries
         A MAST microservice response JSON object (python dictionary)
     data_key : str
         string that contains the key name in json_obj that stores the data rows
@@ -50,6 +50,7 @@ def _json_to_table(json_obj, data_key='data'):
 
     # for each item in info, store the type and column name
     # for each item in info, type has to be converted from DB data types (SQL server in most cases)
+    # from missions_mast search service such as varchar, integer, float, boolean etc
     # to corresponding numpy type
     for idx, col, col_type, ignore_value in \
             [(idx, x['name'], x[type_key].lower(), None) for idx, x in enumerate(json_obj['info'])]:
@@ -78,6 +79,7 @@ def _json_to_table(json_obj, data_key='data'):
             # Step through data array of values
             col_data = np.array([x[idx] for x in json_obj[data_key]], dtype=object)
         except KeyError:
+            # it's not a data array, fall back to using column name as it is array of dictionaries
             col_data = np.array([x[col] for x in json_obj[data_key]], dtype=object)
         if ignore_value is not None:
             col_data[np.where(np.equal(col_data, None))] = ignore_value

astroquery/mast/tests/data/mission_incorrect_results.json

@@ -131,5 +131,5 @@
       }
     ]
   },
-  "totalResults": 3,
+  "totalResults": 3
 }

astroquery/mast/tests/test_mast.py

@@ -1,5 +1,6 @@
 # Licensed under a 3-clause BSD style license - see LICENSE.rst
 
+import json
 import os
 import re
 from shutil import copyfile
@@ -12,6 +13,7 @@
 
 import astropy.units as u
 
+from astroquery.mast.services import _json_to_table
 from astroquery.utils.mocks import MockResponse
 from astroquery.exceptions import InvalidQueryError, InputWarning
 
@@ -258,6 +260,7 @@ def test_missions_query_criteria_async_with_missing_results(patch_post):
                                                                        obs_type,
                                                                        aec,
                                                                        aperture])
+        table = _json_to_table(json.loads(responses), 'results')
 
 
 ###################

docs/mast/mast.rst

@@ -229,7 +229,7 @@ For a non positional search, select_cols would always include search_key and sci
    >>> from astropy.coordinates import SkyCoord
    >>> missions = MastMissions(mission='hst')
    >>> regionCoords = SkyCoord(210.80227, 54.34895, unit=('deg', 'deg'))
-   >>> results = missions.query_region(regionCoords, 3, sci_pep_id=12556,
+   >>> results = missions.query_region(regionCoords, radius=3, sci_pep_id=12556,
    ...                                 select_cols=["sci_stop_time", "sci_targname", "sci_start_time", "sci_status"],
    ...                                 sort_by=['sci_targname'])
    >>> results[:5]   # doctest: +IGNORE_OUTPUT
@@ -254,7 +254,8 @@ of returned records. the default values for offset and limit is 0 and 5000 respe
    >>> missions = MastMissions()
    >>> results = missions.query_criteria(sci_start_time=">=2021-01-01 00:00:00",
    ...                                   select_cols=["sci_stop_time", "sci_targname", "sci_start_time", "sci_status", "sci_pep_id"],
-   ...                                   sort_by=['sci_pep_id'], limit=1000, offset=1000)
+   ...                                   sort_by=['sci_pep_id'], limit=1000, offset=1000)  # doctest: +IGNORE_WARNINGS
+   ... # MaxResultsWarning('Maximum results returned, may not include all sources within radius.')
    >>> len(results)
    1000
 
@@ -263,7 +264,7 @@ Metadata queries can also be performed using object names with the
 
 .. doctest-remote-data::
 
-   >>> results = missions.query_object('M101', 3, select_cols=["sci_stop_time", "sci_targname", "sci_start_time", "sci_status"],
+   >>> results = missions.query_object('M101', radius=3, select_cols=["sci_stop_time", "sci_targname", "sci_start_time", "sci_status"],
    ...                                 sort_by=['sci_targname'])
    >>> results[:5]  # doctest: +IGNORE_OUTPUT
    <Table masked=True length=5>