Merge pull request #3305 from cds-astro/simbad-async-sync

[SIMBAD] add async_job option in all queries

Author: bsipocz

CHANGES.rst

@@ -44,6 +44,13 @@ imcce
 - Changing RuntimeError to NoResultsWarning when an empty result is
   returned. [#3307]
 
+SIMBAD
+^^^^^^
+
+- add ``async_job`` option in all query methods. This executes the query in asynchronous
+  mode. It provides slower to start, but more robust queries for which the timeout can
+  be increased (with the ``timeout`` property or with the configuration file) [#3305]
+
 utils.tap
 ^^^^^^^^^
 

astroquery/simbad/__init__.py

@@ -22,8 +22,11 @@ class Conf(_config.ConfigNamespace):
         'Name of the SIMBAD mirror to use.')
 
     timeout = _config.ConfigItem(
-        60,
-        'Time limit for connecting to Simbad server.')
+        1080,
+        # this is the default value in SIMBAD's main mirror
+        # https://simbad.cds.unistra.fr/simbad/sim-tap/capabilities
+        "Time limit for the execution of asynchronous queries, "
+        "in seconds.")
 
     row_limit = _config.ConfigItem(
         # defaults to the maximum limit

astroquery/simbad/core.py

@@ -8,9 +8,11 @@
 import astropy.units as u
 from astropy.utils.exceptions import AstropyDeprecationWarning
 from pyvo.dal.tap import TAPService
+from pyvo.io.vosi import tapregext
 
 import pytest
 
+from .. import conf
 from ... import simbad
 from .test_simbad_remote import multicoords
 from astroquery.exceptions import NoResultsWarning
@@ -155,6 +157,26 @@ def test_mocked_simbad():
     # and the uploadlimit
     assert simbad_instance.uploadlimit == 200000
 
+
+def test_simbad_timeout(monkeypatch):
+    simbad_instance = simbad.Simbad()
+    assert simbad_instance.timeout == conf.timeout  # default value
+
+    class PatchedCapability:
+        @property
+        def executionduration(self):
+            time_limit = tapregext.TimeLimits()
+            time_limit.hard = 2000
+            return time_limit
+
+    monkeypatch.setattr(TAPService, "capabilities", [PatchedCapability()])
+    # good value
+    simbad_instance.timeout = 10
+    assert simbad_instance.timeout == 10
+    # too high
+    with pytest.raises(ValueError, match="'timeout' cannot exceed*"):
+        simbad_instance.timeout = 10000
+
 # ----------------------------
 # Test output options settings
 # ----------------------------
@@ -563,14 +585,16 @@ def test_query_tap_errors():
 @pytest.mark.usefixtures("_mock_simbad_class")
 def test_query_tap_cache_call(monkeypatch):
     msg = "called_cached_query_tap"
-    monkeypatch.setattr(simbad.core, "_cached_query_tap", lambda tap, query, maxrec: msg)
+    monkeypatch.setattr(simbad.core, "_cached_query_tap",
+                        lambda tap, query, maxrec, async_job, timeout: msg)
     assert simbad.Simbad.query_tap("select top 1 * from basic") == msg
 
 
 @pytest.mark.usefixtures("_mock_simbad_class")
 def test_empty_response_warns(monkeypatch):
     # return something of length zero
-    monkeypatch.setattr(simbad.core.Simbad, "query_tap", lambda _, get_query_payload, maxrec: [])
+    monkeypatch.setattr(simbad.core.Simbad, "query_tap",
+                        lambda _, get_query_payload, maxrec, async_job: [])
     msg = ("The request executed correctly, but there was no data corresponding to these"
            " criteria in SIMBAD")
     with pytest.warns(NoResultsWarning, match=msg):

astroquery/simbad/tests/test_simbad.py

@@ -168,6 +168,12 @@ def test_query_tap(self):
         Simbad.clear_cache()
         assert _cached_query_tap.cache_info().currsize == 0
 
+    def test_async_query(self):
+        adql = "select top 1 main_id from basic"
+        sync_job = Simbad.query_tap(adql)
+        async_job = Simbad.query_tap(adql, async_job=True)
+        assert sync_job["main_id"] == async_job["main_id"]
+
     def test_empty_response_warns(self):
         with pytest.warns(NoResultsWarning, match="The request executed correctly, but *"):
             # a catalog that does not exists should return an empty response

astroquery/simbad/tests/test_simbad_remote.py

@@ -139,22 +139,22 @@ some tables, add their name. To get the columns of the tables ``ref`` and ``bibl
     >>> from astroquery.simbad import Simbad
     >>> Simbad.list_columns("ref", "biblio")
     <Table length=13>
-    table_name column_name   datatype  ...  unit          ucd
-      object      object      object   ... object        object
-    ---------- ----------- ----------- ... ------ --------------------
-        biblio      biblio     VARCHAR ...        meta.record;meta.bib
-        biblio      oidref      BIGINT ...         meta.record;meta.id
-           ref      "year"    SMALLINT ...          meta.note;meta.bib
-           ref    abstract UNICODECHAR ...                 meta.record
-           ref     bibcode        CHAR ...            meta.bib.bibcode
-           ref         doi     VARCHAR ...          meta.code;meta.bib
-           ref     journal     VARCHAR ...            meta.bib.journal
-           ref   last_page     INTEGER ...               meta.bib.page
-           ref    nbobject     INTEGER ...                 meta.number
-           ref      oidbib      BIGINT ...        meta.record;meta.bib
-           ref        page     INTEGER ...               meta.bib.page
-           ref       title UNICODECHAR ...                  meta.title
-           ref      volume     INTEGER ...             meta.bib.volume
+    table_name column_name   datatype  ...  unit         ucd       
+      object      object      object   ... object       object     
+    ---------- ----------- ----------- ... ------ -----------------
+        biblio      biblio     VARCHAR ...         meta.bib.bibcode
+        biblio      oidref      BIGINT ...              meta.record
+           ref      "year"    SMALLINT ...           time.publiYear
+           ref    abstract UNICODECHAR ...              meta.record
+           ref     bibcode        CHAR ...         meta.bib.bibcode
+           ref         doi     VARCHAR ...             meta.ref.doi
+           ref     journal     VARCHAR ...         meta.bib.journal
+           ref   last_page     INTEGER ...            meta.bib.page
+           ref    nbobject     INTEGER ...        meta.id;arith.sum
+           ref      oidbib      BIGINT ...              meta.record
+           ref        page     INTEGER ...            meta.bib.page
+           ref       title UNICODECHAR ...               meta.title
+           ref      volume     INTEGER ...          meta.bib.volume
 
 `~astroquery.simbad.SimbadClass.list_columns` can also be called with a keyword argument.
 This returns columns from any table for witch the given keyword is either in the table name,

docs/simbad/query_tap.rst

@@ -829,6 +829,30 @@ Query TAP
 Troubleshooting
 ===============
 
+Longer queries
+--------------
+
+It can be useful to execute longer queries in asynchronous mode by setting the
+``async_job`` argument to ``True``. This may take longer to start, depending on the
+current number of other people using the asynchronous SIMBAD queue, but it is more
+robust against transient errors. Asynchronous queries will take the ``timeout`` property
+in account:
+
+.. doctest-remote-data::
+
+    >>> from astroquery.simbad import Simbad
+    >>> simbad = Simbad(timeout=2000) # in seconds
+    >>> simbad.query_tap("select otype, description from otypedef where otype = 'N*'",
+    ...                  async_job=True)
+    <Table length=1>
+    otype  description
+    object    object
+    ------ ------------
+        N* Neutron Star
+
+Clearing the cache
+------------------
+
 If you are repeatedly getting failed queries, or bad/out-of-date results, try clearing
 your cache: