astroquery sometimes works fine with query result but most of the time not

[YouCodeLikeAGirl]

  File "C:\Users\..\AppData\Local\Programs\Python\Python36\lib\site-packages\astroquery\utils\class_or_instance.py", line 25, in f
    return self.fn(obj, *args, **kwds)
  File "C:\Users\..\AppData\Local\Programs\Python\Python36\lib\site-packages\astroquery\utils\process_asyncs.py", line 29, in newmethod
    result = self._parse_result(response, verbose=verbose)
  File "C:\Users\..\AppData\Local\Programs\Python\Python36\lib\site-packages\astroquery\simbad\core.py", line 1066, in _parse_result
    "Exception: " + str(ex))
astroquery.exceptions.TableParseError: Failed to parse SIMBAD result! The raw response can be found in self.last_response, and the error in self.last_table_parse_error. The attempted parsed result is in self.last_parsed_result.
 Exception: 7:115: no element found

Sometimes I can receive query result but other times I use exact same query with above error. Could you please investigate the reason and provide a fix?

Thank you in advance.

[bsipocz]

Please post a full example you are trying to run, it would help us reproduce the issue and debug.

[astrobatty]

Hi,

I encountered a similar issue. I'm using python 3.7.6, astropy 4.1 and astroquery 0.4.3.

If I run the following:

from astroquery.simbad import Simbad
from astropy.coordinates import SkyCoord
import astropy.units as u

ra = '21:06:54.0'
dec = '+38:44:58'

result_table = Simbad.query_region(SkyCoord(ra,dec,unit=(u.hourangle, u.deg), frame='icrs'), radius=30*u.arcsec)

I got an error that says there are no results:

ValueError                                Traceback (most recent call last)
~/opt/miniconda3/lib/python3.7/site-packages/astroquery/simbad/core.py in _parse_result(self, result, resultclass, verbose)
   1052                 return None
-> 1053             resulttable = self.last_parsed_result.table
   1054             if len(resulttable) == 0:

~/opt/miniconda3/lib/python3.7/site-packages/astroquery/simbad/core.py in table(self)
    191             self.bytes = BytesIO(self.data.encode('utf8'))
--> 192             tbl = votable.parse_single_table(self.bytes, pedantic=False)
    193             self.__table = tbl.to_table()

~/opt/miniconda3/lib/python3.7/site-packages/astropy/io/votable/table.py in parse_single_table(source, **kwargs)
    184 
--> 185     votable = parse(source, **kwargs)
    186 

~/opt/miniconda3/lib/python3.7/site-packages/astropy/utils/decorators.py in wrapper(*args, **kwargs)
    534 
--> 535             return function(*args, **kwargs)
    536 

~/opt/miniconda3/lib/python3.7/site-packages/astropy/io/votable/table.py in parse(source, columns, invalid, verify, chunk_size, table_number, table_id, filename, unit_format, datatype_mapping, _debug_python_based_parser)
    166         return tree.VOTableFile(
--> 167             config=config, pos=(1, 1)).parse(iterator, config)
    168 

~/opt/miniconda3/lib/python3.7/site-packages/astropy/io/votable/tree.py in parse(self, iterator, config)
   3574                 tag_mapping.get(tag, self._add_unknown_tag)(
-> 3575                     iterator, tag, data, config, pos)
   3576             elif tag == 'DESCRIPTION':

~/opt/miniconda3/lib/python3.7/site-packages/astropy/io/votable/tree.py in _add_resource(self, iterator, tag, data, config, pos)
   3484         self.resources.append(resource)
-> 3485         resource.parse(self, iterator, config)
   3486 

~/opt/miniconda3/lib/python3.7/site-packages/astropy/io/votable/tree.py in parse(self, votable, iterator, config)
   3286                 tag_mapping.get(tag, self._add_unknown_tag)(
-> 3287                     iterator, tag, data, config, pos)
   3288             elif tag == 'DESCRIPTION':

~/opt/miniconda3/lib/python3.7/site-packages/astropy/io/votable/tree.py in _add_table(self, iterator, tag, data, config, pos)
   3231         self.tables.append(table)
-> 3232         table.parse(iterator, config)
   3233 

~/opt/miniconda3/lib/python3.7/site-packages/astropy/io/votable/tree.py in parse(self, iterator, config)
   2451 
-> 2452             for start, tag, data, pos in iterator:
   2453                 if start:

ValueError: 7:115: no element found

During handling of the above exception, another exception occurred:

TableParseError                           Traceback (most recent call last)
<ipython-input-10-bf9c439272eb> in <module>
----> 1 result_table = Simbad.query_region(SkyCoord(ra,dec,unit=(u.hourangle, u.deg), frame='icrs'), radius=30*u.arcsec)

~/opt/miniconda3/lib/python3.7/site-packages/astroquery/utils/class_or_instance.py in f(*args, **kwds)
     23         def f(*args, **kwds):
     24             if obj is not None:
---> 25                 return self.fn(obj, *args, **kwds)
     26             else:
     27                 return self.fn(cls, *args, **kwds)

~/opt/miniconda3/lib/python3.7/site-packages/astroquery/utils/process_asyncs.py in newmethod(self, *args, **kwargs)
     27             if kwargs.get('get_query_payload') or kwargs.get('field_help'):
     28                 return response
---> 29             result = self._parse_result(response, verbose=verbose)
     30             self.table = result
     31             return result

~/opt/miniconda3/lib/python3.7/site-packages/astroquery/simbad/core.py in _parse_result(self, result, resultclass, verbose)
   1062                                   " parsed result is in "
   1063                                   "self.last_parsed_result.\n "
-> 1064                                   "Exception: " + str(ex))
   1065         resulttable.errors = self.last_parsed_result.errors
   1066         return resulttable

TableParseError: Failed to parse SIMBAD result! The raw response can be found in self.last_response, and the error in self.last_table_parse_error. The attempted parsed result is in self.last_parsed_result.
 Exception: 7:115: no element found

However, If I extend the radius from 30" to 40", everything starts to work fine:

result_table = Simbad.query_region(SkyCoord(ra,dec,unit=(u.hourangle, u.deg), frame='icrs'), radius=40*u.arcsec)

result_table

The problem is that my target is well within my first search radius, but somehow not found. The true distance is less than 1":

from astropy.coordinates import match_coordinates_sky

c_in = SkyCoord(ra,dec,unit=(u.hourangle, u.deg), frame='icrs')
c_out = SkyCoord(result_table['RA'],result_table['DEC'],unit=(u.hourangle, u.deg), frame='icrs')

match_coordinates_sky(c_in,c_out)[1].arcsec

array([0.59340467])

If I perform the same query on the Simbad webpage with 30' I get the first two targets which are within 30". Maybe something is wrong the handling of astropy units?

[keflavich]

First, you've written 30' but your code says 30". Which did you mean?

I was able to reproduce this error once, but clearing the cache and re-running solved it.

Original error:

Traceback (most recent call last):
  File "<ipython-input-47-62263cad87ae>", line 1, in <module>
    Simbad.query_region(SkyCoord(ra,dec,unit=(u.hourangle, u.deg), frame='icrs'), radius=30*u.arcsec)
  File "/Users/adam/repos/astroquery/astroquery/utils/class_or_instance.py", line 25, in f
    return self.fn(obj, *args, **kwds)
  File "/Users/adam/repos/astroquery/astroquery/utils/process_asyncs.py", line 29, in newmethod
    result = self._parse_result(response, verbose=verbose)
  File "/Users/adam/repos/astroquery/astroquery/simbad/core.py", line 1064, in _parse_result
    "Exception: " + str(ex))
TableParseError: Failed to parse SIMBAD result! The raw response can be found in self.last_response, and the error in self.last_table_parse_error. The attempted parsed result is in self.last_parsed_result.
 Exception: 7:115: no element found

>>> Simbad.query_region(SkyCoord(ra,dec,unit=(u.hourangle, u.deg), frame='icrs'), radius=30*u.arcsec, cache=False)

<Table length=2>
  MAIN_ID         RA           DEC      RA_PREC DEC_PREC COO_ERR_MAJA COO_ERR_MINA COO_ERR_ANGLE COO_QUAL COO_WAVELENGTH     COO_BIBCODE     SCRIPT_NUMBER_ID
               "h:m:s"       "d:m:s"                         mas          mas           deg
   object       str13         str13      int16   int16     float32      float32        int16       str1        str1             object            int32
----------- ------------- ------------- ------- -------- ------------ ------------ ------------- -------- -------------- ------------------- ----------------
  *  61 Cyg   21 06 53.95   +38 44 57.9       6        6           --           --             0        D              O 2001AJ....122.3466M                1
*  61 Cyg A 21 06 53.9396 +38 44 57.897      14       14        0.073        0.086            90        A              O 2018yCat.1345....0G                1

Clearing the cache is... still something we need to document better.

rm ~/.astropy/cache/astroquery/Simbad/*.pickle

[bsipocz]

@keflavich - I suppose we could rewrite the error message and raise a NoResultsWarning instead as we do in a few other modules?

[keflavich]

Perhaps? But I'm not entirely sure what has caused this error; it seems to be an intermittent upstream error.

There is perhaps a real bug here: a failed query should not be cached, and this is a failed query, not just a "no data found" query.

[bsipocz]

Oh, sure if it's an upstream error, we still need to parse it more gently, and surely don't cache it. At minimum more examples are needed to how to handle cache and these type of issues.

Relevant issues: #1838 also, there are a few cache related PRs and issues that maybe relevant: e.g. would be nice to wrap up #1634 and company.

[keflavich]

OK, I was able to reproduce with a few tests. This is an example of a failed query:

::script::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

votable {main_id,coordinates}
votable open
query coo 21:06:54 +38:44:58 radius=5.0s frame=ICRS equi=2000.0
votable close

::console:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

C.D.S.  -  SIMBAD4 rel 1.7  -  2021.10.24CEST23:06:32
total execution time: 0.073 secs
simbatch done

::data::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

<?xml version="1.0" encoding="UTF-8"?>
<VOTABLE xmlns="http://www.ivoa.net/xml/VOTable/v1.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="1.2" ID="http://www.ivoa.net/xml/VOTable/v1.2 http://www.ivoa.net/xml/VOTable/v1.2">
<DEFINITIONS>
<COOSYS ID="COOSYS" equinox="2000" epoch="J2000" system="ICRS"/>
</DEFINITIONS>
<RESOURCE name="Simbad query" type="results">
<TABLE ID="SimbadScript" name="default"><DESCRIPTION>Simbad script executed on 2021.10.24CEST23:06:32</DESCRIPTION>

and a successful one:

::script::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

votable {main_id,coordinates}
votable open
query coo 21:06:54 +38:44:58 radius=5.0s frame=ICRS equi=2000.0
votable close

::console:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

C.D.S.  -  SIMBAD4 rel 1.7  -  2021.10.24CEST23:09:11
total execution time: 0.069 secs
simbatch done

::data::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

<?xml version="1.0" encoding="UTF-8"?>
<VOTABLE xmlns="http://www.ivoa.net/xml/VOTable/v1.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="1.2" ID="http://www.ivoa.net/xml/VOTable/v1.2 http://www.ivoa.net/xml/VOTable/v1.2">
<DEFINITIONS>
<COOSYS ID="COOSYS" equinox="2000" epoch="J2000" system="ICRS"/>
</DEFINITIONS>
<RESOURCE name="Simbad query" type="results">
<TABLE ID="SimbadScript" name="default"><DESCRIPTION>Simbad script executed on 2021.10.24CEST23:09:11</DESCRIPTION>
<FIELD ID="MAIN_ID" name="MAIN_ID" datatype="char" width="22" ucd="meta.id;meta.main" arraysize="*">
<DESCRIPTION>Main identifier for an object</DESCRIPTION>
<LINK value="${MAIN_ID}" href="http://simbad.u-strasbg.fr/simbad/sim-id?Ident=${MAIN_ID}&amp;NbIdent=1"/>
</FIELD>
<FIELD ID="RA" name="RA" datatype="char" precision="8" width="13" ucd="pos.eq.ra;meta.main" arraysize="13" unit="&quot;h:m:s&quot;">
<DESCRIPTION>Right ascension</DESCRIPTION>
</FIELD>
<FIELD ID="DEC" name="DEC" datatype="char" precision="8" width="13" ucd="pos.eq.dec;meta.main" arraysize="13" unit="&quot;d:m:s&quot;">
<DESCRIPTION>Declination</DESCRIPTION>
</FIELD>
<FIELD ID="RA_PREC" name="RA_PREC" datatype="short" width="2">
<DESCRIPTION>Right ascension precision</DESCRIPTION>
</FIELD>
<FIELD ID="DEC_PREC" name="DEC_PREC" datatype="short" width="2">
<DESCRIPTION>Declination precision</DESCRIPTION>
</FIELD>
<FIELD ID="COO_ERR_MAJA" name="COO_ERR_MAJA" datatype="float" precision="3" width="6" ucd="phys.angSize.smajAxis;pos.errorEllipse;pos.eq" unit="mas">
<DESCRIPTION>Coordinate error major axis</DESCRIPTION>
</FIELD>
<FIELD ID="COO_ERR_MINA" name="COO_ERR_MINA" datatype="float" precision="3" width="6" ucd="phys.angSize.sminAxis;pos.errorEllipse;pos.eq" unit="mas">
<DESCRIPTION>Coordinate error minor axis</DESCRIPTION>
</FIELD>
<FIELD ID="COO_ERR_ANGLE" name="COO_ERR_ANGLE" datatype="short" width="3" ucd="pos.posAng;pos.errorEllipse;pos.eq" unit="deg">
<DESCRIPTION>Coordinate error angle</DESCRIPTION>
</FIELD>
<FIELD ID="COO_QUAL" name="COO_QUAL" datatype="char" width="1" ucd="meta.code.qual;pos.eq">
<DESCRIPTION>Coordinate quality</DESCRIPTION>
</FIELD>
<FIELD ID="COO_WAVELENGTH" name="COO_WAVELENGTH" datatype="char" width="1" ucd="instr.bandpass;pos.eq">
<DESCRIPTION>Wavelength class for the origin of the coordinates (R,I,V,U,X,G)</DESCRIPTION>
</FIELD>
<FIELD ID="COO_BIBCODE" name="COO_BIBCODE" datatype="char" width="19" ucd="meta.bib.bibcode;pos.eq" arraysize="*">
<DESCRIPTION>Coordinate reference</DESCRIPTION>
</FIELD>
<FIELD ID="SCRIPT_NUMBER_ID" name="SCRIPT_NUMBER_ID" datatype="int" precision="1" width="2" ucd="meta.number"/>
<DATA>
<TABLEDATA>
<TR><TD>*  61 Cyg</TD><TD>21 06 53.95</TD><TD>+38 44 57.9</TD><TD>6</TD><TD>6</TD><TD></TD><TD></TD><TD></TD><TD>D</TD><TD>Opt</TD><TD>2001AJ....122.3466M</TD><TD>1</TD></TR>
<TR><TD>*  61 Cyg A</TD><TD>21 06 53.9396100677</TD><TD>+38 44 57.897024357</TD><TD>14</TD><TD>14</TD><TD>0.0730</TD><TD>0.0863</TD><TD>90</TD><TD>A</TD><TD>Opt</TD><TD>2018yCat.1345....0G</TD><TD>1</TD></TR>

</TABLEDATA>

</DATA>
</TABLE>
</RESOURCE>
</VOTABLE>

so it looks like if there is no </VOTABLE>, the query has failed?

[keflavich]

(note that I was able to reproduce the error through the SIMBAD sim-script form - this isn't an astroquery issue, but we can handle it better)

[astrobatty]

First, you've written 30' but your code says 30". Which did you mean?

I was able to reproduce this error once, but clearing the cache and re-running solved it.

Original error:

Traceback (most recent call last):
  File "<ipython-input-47-62263cad87ae>", line 1, in <module>
    Simbad.query_region(SkyCoord(ra,dec,unit=(u.hourangle, u.deg), frame='icrs'), radius=30*u.arcsec)
  File "/Users/adam/repos/astroquery/astroquery/utils/class_or_instance.py", line 25, in f
    return self.fn(obj, *args, **kwds)
  File "/Users/adam/repos/astroquery/astroquery/utils/process_asyncs.py", line 29, in newmethod
    result = self._parse_result(response, verbose=verbose)
  File "/Users/adam/repos/astroquery/astroquery/simbad/core.py", line 1064, in _parse_result
    "Exception: " + str(ex))
TableParseError: Failed to parse SIMBAD result! The raw response can be found in self.last_response, and the error in self.last_table_parse_error. The attempted parsed result is in self.last_parsed_result.
 Exception: 7:115: no element found

>>> Simbad.query_region(SkyCoord(ra,dec,unit=(u.hourangle, u.deg), frame='icrs'), radius=30*u.arcsec, cache=False)

<Table length=2>
  MAIN_ID         RA           DEC      RA_PREC DEC_PREC COO_ERR_MAJA COO_ERR_MINA COO_ERR_ANGLE COO_QUAL COO_WAVELENGTH     COO_BIBCODE     SCRIPT_NUMBER_ID
               "h:m:s"       "d:m:s"                         mas          mas           deg
   object       str13         str13      int16   int16     float32      float32        int16       str1        str1             object            int32
----------- ------------- ------------- ------- -------- ------------ ------------ ------------- -------- -------------- ------------------- ----------------
  *  61 Cyg   21 06 53.95   +38 44 57.9       6        6           --           --             0        D              O 2001AJ....122.3466M                1
*  61 Cyg A 21 06 53.9396 +38 44 57.897      14       14        0.073        0.086            90        A              O 2018yCat.1345....0G                1

Clearing the cache is... still something we need to document better.

rm ~/.astropy/cache/astroquery/Simbad/*.pickle

Sorry, I meant 30", but as I can see you've already found the source of problem.

[bsipocz]

I think this has been resolved in #2187. Regarding the need for better documentation for the caching, it already has issues opened, so for that purpose, there is no need to keep this one open, too.

[emirkmo]

rm ~/.astropy/cache/astroquery/Simbad/*.pickle

Ran into a similar issue that relied on an older version of Astropy (v4.3) and this is what solved it, thanks.