Merge branch 'astropy:main' into feature/ISSUE-3138-TAP

Author: juanmcloaiza

CHANGES.rst

@@ -19,6 +19,8 @@ gaia
 
 - Deprecated ``band`` from ``load_data`` as it has no effect on upstream
   response any more. [#3278]
+- New datalink retrieve types EPOCH_PHOTOMETRY_CROWDED_FIELD, EPOCH_ASTROMETRY_BRIGHT, XP_MEAN_SPECTRUM_GRAVLENS,
+  EPOCH_FLAGS_NSS, EPOCH_PARAMETERS_RVS_SINGLE, EPOCH_PARAMETERS_RVS_DOUBLE, EPOCH_FLAGS_VARI. [#3371, #3372]
 
 Service fixes and enhancements
 ------------------------------
@@ -48,6 +50,7 @@ gaia
 ^^^^
 
 - New method cross_match_basic that simplifies the positional x-match method [#3320]
+- new DR4 datalink retrieve type MEAN_SPECTRUM_RVS [#3342]
 
 linelists.cdms
 ^^^^^^^^^^^^^^
@@ -60,6 +63,7 @@ heasarc
 - Add support for astropy.table.Row in Heasarc.download_data and Heasarc.locate_data. [#3270]
 - Heasarc.locate_data returns empty rows with an error in the error_message column if there are
   no data associated with that row rather than filtering it out. [#3275]
+- Heasarc.locate_data changed to use POST request instead of GET to accomodate large requests. [#3356]
 
 imcce
 ^^^^^
@@ -70,6 +74,14 @@ imcce
 - Changing RuntimeError to NoResultsWarning when an empty result is
   returned. [#3307]
 
+ipac.irsa
+^^^^^^^^^
+
+- Fix ``list_catalogs`` to not include image metadata tables, only
+  catalogs. The ``include_metadata_tables`` keyword argument allows opting
+  in to return all TAP tables, including non-spatial and metadata ones,
+  too. [#3334]
+
 SIMBAD
 ^^^^^^
 
@@ -121,6 +133,12 @@ query.py
   Some corner cases where downloads were not properly continued have been
   fixed. [#3232]
 
+skyview
+^^^^^^^
+
+- Changed SkyView URL to https. [#3346]
+
+
 utils
 ^^^^^
 

astroquery/gaia/__init__.py

@@ -29,6 +29,7 @@ class Conf(_config.ConfigNamespace):
                                       'XP_CONTINUOUS',
                                       'XP_SAMPLED',
                                       'RVS',
+                                      'MEAN_SPECTRUM_RVS',
                                       'MCMC_GSPPHOT',
                                       'MCMC_MSC',
                                       'EPOCH_ASTROMETRY',
@@ -37,14 +38,20 @@ class Conf(_config.ConfigNamespace):
                                       'RVS_EPOCH_SPECTRUM',
                                       'RVS_TRANSIT',
                                       'EPOCH_ASTROMETRY_CROWDED_FIELD',
+                                      'EPOCH_PHOTOMETRY_CROWDED_FIELD',
                                       'EPOCH_IMAGE',
                                       'EPOCH_PHOTOMETRY_CCD',
                                       'XP_EPOCH_SPECTRUM_SSO',
                                       'XP_EPOCH_CROWDING',
                                       'XP_MEAN_SPECTRUM',
                                       'XP_EPOCH_SPECTRUM',
                                       'CROWDED_FIELD_IMAGE',
-                                      'EPOCH_ASTROMETRY_BRIGHT']
+                                      'EPOCH_ASTROMETRY_BRIGHT',
+                                      'XP_MEAN_SPECTRUM_GRAVLENS',
+                                      'EPOCH_FLAGS_NSS',
+                                      'EPOCH_PARAMETERS_RVS_SINGLE',
+                                      'EPOCH_PARAMETERS_RVS_DOUBLE',
+                                      'EPOCH_FLAGS_VARI']
 
     VALID_LINKING_PARAMETERS = {'SOURCE_ID', 'TRANSIT_ID', 'IMAGE_ID'}
 

astroquery/gaia/core.py

@@ -184,18 +184,20 @@ def load_data(self, ids, *, data_release=None, data_structure='INDIVIDUAL', retr
             in a single bundle, even if only one source/file is considered
             'RAW' means products are provided following a Data Model similar to that used in the MDB, meaning in
             particular that parameters stored as arrays will remain as such. A single file is provided for the data of
-            all sourceIds together, but in this case there will be always be one row per sourceId
+            all sourceIds together, but in this case there will be always be one row per sourceId.
         retrieval_type : str, optional, default 'ALL' to retrieve all data  from the list of sources
             retrieval type identifier. For GAIA DR2 possible values are ['EPOCH_PHOTOMETRY']
             For GAIA DR3, possible values are ['EPOCH_PHOTOMETRY', 'RVS', 'XP_CONTINUOUS', 'XP_SAMPLED',
             'MCMC_GSPPHOT' or 'MCMC_MSC']
-            For GAIA DR4, possible values will be ['EPOCH_PHOTOMETRY', 'RVS', 'XP_CONTINUOUS', 'XP_SAMPLED',
-            'MCMC_GSPPHOT', 'MCMC_MSC', 'EPOCH_ASTROMETRY', 'RVS_EPOCH_DATA_SINGLE',
+            For GAIA DR4, possible values will be ['EPOCH_PHOTOMETRY', 'MEAN_SPECTRUM_RVS', 'XP_CONTINUOUS',
+            'XP_SAMPLED', 'MCMC_GSPPHOT', 'MCMC_MSC', 'EPOCH_ASTROMETRY', 'RVS_EPOCH_DATA_SINGLE',
             'RVS_EPOCH_DATA_DOUBLE','RVS_EPOCH_SPECTRUM', 'RVS_TRANSIT', 'EPOCH_ASTROMETRY_CROWDED_FIELD',
-            'EPOCH_IMAGE', 'EPOCH_PHOTOMETRY_CCD', 'XP_EPOCH_SPECTRUM_SSO', 'XP_EPOCH_CROWDING', 'XP_MEAN_SPECTRUM',
-            'XP_EPOCH_SPECTRUM', 'CROWDED_FIELD_IMAGE', 'EPOCH_ASTROMETRY_BRIGHT']. Note that for
-            'CROWDED_FIELD_IMAGE' only the format 'fits' can be used, and that its image, in the principal header, will
-            not be available in the returned dictionary. Set 'output_file' to retrieve all data: image + tables.
+            'EPOCH_PHOTOMETRY_CROWDED_FIELD', 'EPOCH_IMAGE', 'EPOCH_PHOTOMETRY_CCD', 'XP_EPOCH_SPECTRUM_SSO',
+            'XP_EPOCH_CROWDING', 'XP_MEAN_SPECTRUM', 'XP_EPOCH_SPECTRUM', 'CROWDED_FIELD_IMAGE',
+            'EPOCH_ASTROMETRY_BRIGHT', 'XP_MEAN_SPECTRUM_GRAVLENS', 'EPOCH_FLAGS_NSS', 'EPOCH_PARAMETERS_RVS_SINGLE',
+            EPOCH_PARAMETERS_RVS_DOUBLE', 'EPOCH_FLAGS_VARI']. Note that for 'CROWDED_FIELD_IMAGE' only the format
+            'fits' can be used, and that its image, in the principal header, will not be available in the returned
+            dictionary. Set 'output_file' to retrieve all data: image + tables.
         linking_parameter : str, optional, default SOURCE_ID, valid values: SOURCE_ID, TRANSIT_ID, IMAGE_ID
             By default, all the identifiers are considered as source_id
             SOURCE_ID: the identifiers are considered as source_id

astroquery/heasarc/core.py

@@ -517,13 +517,18 @@ def locate_data(self, query_result=None, catalog_name=None):
 
         # datalink url
         dlink_url = f'{self.VO_URL}/datalink/{catalog_name}'
-
         query = pyvo.dal.adhoc.DatalinkQuery(
             baseurl=dlink_url,
             id=query_result['__row'],
             session=self._session
         )
-        dl_result = query.execute().to_table()
+
+        dl_result = pyvo.dal.DALResults(
+            query.execute_votable(post=True),
+            url=query.queryurl,
+            session=query._session
+        ).to_table()
+
         # include rows that have directory links (i.e. data) and those
         # that report errors (usually means there are no data products)
         dl_result = dl_result[np.ma.mask_or(

astroquery/ipac/irsa/core.py

@@ -304,7 +304,7 @@ def query_region(self, coordinates=None, *, catalog=None, spatial='Cone',
         return response.to_table()
 
     @deprecated_renamed_argument("cache", None, since="0.4.7")
-    def list_catalogs(self, *, full=False, filter=None, cache=False):
+    def list_catalogs(self, *, full=False, filter=None, include_metadata_tables=False, cache=False):
         """
         Return information of available IRSA catalogs.
 
@@ -316,8 +316,19 @@ def list_catalogs(self, *, full=False, filter=None, cache=False):
         filter : str or None
             If specified we only return catalogs when their catalog_name
             contains the filter string.
+        include_metadata_tables : bool
+            If True returns not just the catalogs but all table holdings including the image metadata tables.
+            These are not suitable for spatial queries with e.g. ``query_region``.
         """
-        tap_tables = self.query_tap("SELECT * FROM TAP_SCHEMA.tables").to_table()
+
+        if include_metadata_tables:
+            more_filtering = ""
+        else:
+            # Filter out non-spatial catalogs and metadata tables with
+            # irsa_pos=y and irsa_dbms=21
+            more_filtering = "WHERE irsa_dbms=21 AND irsa_pos='y'"
+
+        tap_tables = self.query_tap(f"SELECT * FROM TAP_SCHEMA.tables {more_filtering}").to_table()
 
         if filter:
             mask = [filter in name for name in tap_tables['table_name']]

astroquery/ipac/irsa/tests/test_irsa_remote.py

@@ -72,14 +72,26 @@ def test_list_columns(self):
     def test_list_catalogs(self):
         catalogs = Irsa.list_catalogs()
         # Number of available catalogs may change over time, test only for significant drop.
-        # (at the time of writing there are 933 tables in the list).
-        assert len(catalogs) > 900
+        # (at the time of writing there are 521 catalogs in the list).
+        assert len(catalogs) > 520
         assert isinstance(catalogs, dict)
 
+        catalogs_full = Irsa.list_catalogs(full=True)
+        assert isinstance(catalogs_full, Table)
+
     def test_list_catalogs_filter(self):
-        spitzer_catalogs = Irsa.list_catalogs(filter='spitzer')
+        allwise_catalogs = Irsa.list_catalogs(filter='allwise')
+
+        assert len(allwise_catalogs) == 4
+
+    def test_list_catalogs_metadata(self):
+        catalogs = Irsa.list_catalogs(filter='wise')
+        all_tables = Irsa.list_catalogs(filter='wise', include_metadata_tables=True)
+
+        assert len(catalogs) < len(all_tables)
 
-        assert len(spitzer_catalogs) == 142
+        assert 'wise.wise_allwise_p3am_cdd' not in catalogs
+        assert 'wise.wise_allwise_p3am_cdd' in all_tables
 
     @pytest.mark.parametrize('servicetype', (None, 'sia', 'ssa'))
     def test_list_collections(self, servicetype):

astroquery/mast/missions.py

@@ -188,19 +188,16 @@ def query_region_async(self, coordinates, *, radius=3*u.arcmin, limit=5000, offs
         coordinates : str or `~astropy.coordinates` object
             The target around which to search. It may be specified as a
             string or as the appropriate `~astropy.coordinates` object.
-        radius : str or `~astropy.units.Quantity` object, optional
-            Default 3 degrees.
+        radius : str or `~astropy.units.Quantity` object
+            Default is 3 arcminutes. The radius around the coordinates to search within.
             The string must be parsable by `~astropy.coordinates.Angle`. The
-            appropriate `~astropy.units.Quantity` object from
-            `~astropy.units` may also be used. Defaults to 3 arcminutes.
+            appropriate `~astropy.units.Quantity` object from `~astropy.units` may also be used.
         limit : int
-            Optional and default is 5000.
-            the maximum number of dataset IDs in the results.
+            Default is 5000. The maximum number of dataset IDs in the results.
         offset : int
-            Optional and default is 0
-            the number of records you wish to skip before selecting records.
-        select_cols: list, None
-            Default None. Names of columns that will be included in the result table.
+            Default is 0. The number of records you wish to skip before selecting records.
+        select_cols: list, optional
+            Default is None. Names of columns that will be included in the result table.
             If None, a default set of columns will be returned.
         **criteria
             Other mission-specific criteria arguments.
@@ -224,7 +221,7 @@ def query_region_async(self, coordinates, *, radius=3*u.arcmin, limit=5000, offs
         # Put coordinates and radius into consistent format
         coordinates = commons.parse_coordinates(coordinates)
 
-        # if radius is just a number we assume degrees
+        # If radius is just a number, assume arcminutes
         radius = coord.Angle(radius, u.arcmin)
 
         # Dataset ID column should always be returned
@@ -233,7 +230,7 @@ def query_region_async(self, coordinates, *, radius=3*u.arcmin, limit=5000, offs
         elif self.mission == 'ullyses':
             select_cols = self._default_ullyses_cols
 
-        # basic params
+        # Basic params
         params = {'target': [f"{coordinates.ra.deg} {coordinates.dec.deg}"],
                   'radius': radius.arcsec,
                   'radius_units': 'arcseconds',
@@ -258,23 +255,20 @@ def query_criteria_async(self, *, coordinates=None, objectname=None, radius=3*u.
             string or as the appropriate `~astropy.coordinates` object.
         objectname : str
             The name of the target around which to search.
-        radius : str or `~astropy.units.Quantity` object, optional
-            Default 3 degrees.
+        radius : str or `~astropy.units.Quantity` object
+            Default is 3 arcminutes. The radius around the coordinates to search within.
             The string must be parsable by `~astropy.coordinates.Angle`. The
-            appropriate `~astropy.units.Quantity` object from
-            `~astropy.units` may also be used. Defaults to 3 arcminutes.
+            appropriate `~astropy.units.Quantity` object from `~astropy.units` may also be used.
         limit : int
-            Optional and default is 5000.
-            the maximum number of dataset IDs in the results.
+            Default is 5000. The maximum number of dataset IDs in the results.
         offset : int
-            Optional and default is 0.
-            the number of records you wish to skip before selecting records.
-        select_cols: list, None
-            Default None. Names of columns that will be included in the result table.
+            Default is 0. The number of records you wish to skip before selecting records.
+        select_cols: list, optional
+            Default is None. Names of columns that will be included in the result table.
             If None, a default set of columns will be returned.
         resolver : str, optional
-            The resolver to use when resolving a named target into coordinates. Valid options are "SIMBAD" and "NED".
-            If not specified, the default resolver order will be used. Please see the
+            Default is None. The resolver to use when resolving a named target into coordinates. Valid options are
+            "SIMBAD" and "NED". If not specified, the default resolver order will be used. Please see the
             `STScI Archive Name Translation Application (SANTA) <https://mastresolver.stsci.edu/Santa-war/>`__
             for more information. Default is None.
         **criteria
@@ -339,22 +333,19 @@ def query_object_async(self, objectname, *, radius=3*u.arcmin, limit=5000, offse
         objectname : str
             The name of the target around which to search.
         radius : str or `~astropy.units.Quantity` object, optional
-            Default 3 arcmin.
-            The string must be parsable by `~astropy.coordinates.Angle`.
-            The appropriate `~astropy.units.Quantity` object from
-            `~astropy.units` may also be used. Defaults to 3 arcminutes.
+            Default is 3 arcminutes. The radius around the coordinates to search within.
+            The string must be parsable by `~astropy.coordinates.Angle`. The
+            appropriate `~astropy.units.Quantity` object from `~astropy.units` may also be used.
         limit : int
-            Optional and default is 5000.
-            the maximum number of dataset IDs in the results.
+            Default is 5000. The maximum number of dataset IDs in the results.
         offset : int
-            Optional and default is 0.
-            the number of records you wish to skip before selecting records.
-        select_cols: list, None
-            Default None. Names of columns that will be included in the result table.
+            Default is 0. The number of records you wish to skip before selecting records.
+        select_cols: list, optional
+            Default is None. Names of columns that will be included in the result table.
             If None, a default set of columns will be returned.
         resolver : str, optional
-            The resolver to use when resolving a named target into coordinates. Valid options are "SIMBAD" and "NED".
-            If not specified, the default resolver order will be used. Please see the
+            Default is None. The resolver to use when resolving a named target into coordinates. Valid options are
+            "SIMBAD" and "NED". If not specified, the default resolver order will be used. Please see the
             `STScI Archive Name Translation Application (SANTA) <https://mastresolver.stsci.edu/Santa-war/>`__
             for more information. Default is None.
         **criteria

astroquery/skyview/__init__.py

@@ -7,7 +7,7 @@ class Conf(_config.ConfigNamespace):
     Configuration parameters for `astroquery.skyview`.
     """
     url = _config.ConfigItem(
-        'http://skyview.gsfc.nasa.gov/current/cgi/basicform.pl',
+        'https://skyview.gsfc.nasa.gov/current/cgi/basicform.pl',
         'SkyView URL')
 
 

astroquery/skyview/tests/data/survey_dict.json

@@ -2,7 +2,7 @@
  "HardX-ray": ["INT GAL 17-35 Flux", "INT GAL 17-60 Flux", "INT GAL 35-80 Flux", "INTEGRAL/SPI GC", "GRANAT/SIGMA", "RXTE Allsky 3-8keV Flux", "RXTE Allsky 3-20keV Flux", "RXTE Allsky 8-20keV Flux"],
  "X-ray:SwiftBAT": ["BAT SNR 14-195", "BAT SNR 14-20", "BAT SNR 20-24", "BAT SNR 24-35", "BAT SNR 35-50", "BAT SNR 50-75", "BAT SNR 75-100", "BAT SNR 100-150", "BAT SNR 150-195"],
  "UV": ["GALEX Near UV", "GALEX Far UV", "ROSAT WFC F1", "ROSAT WFC F2", "EUVE 83 A", "EUVE 171 A", "EUVE 405 A", "EUVE 555 A"],
- "Optical:SDSS": ["SDSSg", "SDSSi", "SDSSr", "SDSSu", "SDSSz", "SDSSdr7g", "SDSSdr7i", "SDSSdr7r", "SDSSdr7u", "SDSSdr7z"],
+ "Optical:SDSS class=": ["SDSSg", "SDSSi", "SDSSr", "SDSSu", "SDSSz"],
  "OtherOptical": ["TESS", "Mellinger Red", "Mellinger Green", "Mellinger Blue", "H-Alpha Comp", "SHASSA H", "SHASSA CC", "SHASSA C", "SHASSA Sm"],
  "IR:IRAS": ["IRIS  12", "IRIS  25", "IRIS  60", "IRIS 100", "SFD100m", "SFD Dust Map", "IRAS  12 micron", "IRAS  25 micron", "IRAS  60 micron", "IRAS 100 micron"],
  "IR:Planck": ["Planck 857 I", "Planck 545 I", "Planck 353 I", "Planck 353 Q", "Planck 353 U", "Planck 353 PI", "Planck 353 PA", "Planck 353 PI/I", "Planck 217 I", "Planck 217 Q", "Planck 217 U", "Planck 217 PI", "Planck 217 PA", "Planck 217 PI/I", "Planck 143 I", "Planck 143 Q", "Planck 143 U", "Planck 143 PI", "Planck 143 PA", "Planck 143 PI/I", "Planck 100 I", "Planck 100 Q", "Planck 100 U", "Planck 100 PI", "Planck 100 PA", "Planck 100 PI/I", "Planck 070 I", "Planck 070 Q", "Planck 070 U", "Planck 070 PI", "Planck 070 PA", "Planck 070 PI/I", "Planck 044 I", "Planck 044 Q", "Planck 044 U", "Planck 044 PI", "Planck 044 PA", "Planck 044 PI/I", "Planck 030 I", "Planck 030 Q", "Planck 030 U", "Planck 030 PI", "Planck 030 PA", "Planck 030 PI/I"],

astroquery/splatalogue/core.py

@@ -219,7 +219,7 @@ def _parse_kwargs(self, *, min_frequency=None, max_frequency=None,
             cm/K for *inverse* cm, i.e. wavenumber, or K for Kelvin
         intensity_lower_limit : `None` or float
             Lower limit on the intensity.  See intensity_type
-        intensity_type : `None` or ``'sij'``, ``'cdms_jpl'``, ``'aij'``
+        intensity_type : `None`, ``'CDMS/JPL (log)'``, ``'Sij-mu2'``, ``'Aij (log)'``
             The type of intensity on which to place a lower limit
         transition : str
             e.g. 1-0