Merge pull request #1375 from bsipocz/xmatch_votable

Switching to use VOtable format for XMatch

Author: bsipocz

CHANGES.rst

@@ -186,6 +186,12 @@ mast
 - Changed warning to error for authentication failure. [#1874]
 
 
+xmatch
+^^^^^^
+
+- Minor internal change to use VOTable as the response format that include
+  units, too. [#1375]
+
 
 Infrastructure, Utility and Other Changes and Additions
 -------------------------------------------------------

astroquery/xmatch/core.py

@@ -2,12 +2,11 @@
 
 from io import StringIO, BytesIO
 
-from astropy.io import ascii, votable
+from astropy.io import votable
 import astropy.units as u
 from astropy.table import Table
 from requests import HTTPError
 
-
 from astroquery.query import BaseQuery
 from astroquery.exceptions import InvalidQueryError
 from astroquery.utils import url_helpers, prepend_docstr_nosections, async_to_sync
@@ -77,7 +76,9 @@ def query(self, cat1, cat2, max_distance,
                                     **kwargs)
         if get_query_payload:
             return response
-        return self._parse_text(response.text)
+
+        content = BytesIO(response.content)
+        return Table.read(content, format='votable', use_names_over_ids=True)
 
     @prepend_docstr_nosections("\n" + query.__doc__)
     def query_async(self, cat1, cat2, max_distance, colRA1=None, colDec1=None,
@@ -92,12 +93,11 @@ def query_async(self, cat1, cat2, max_distance, colRA1=None, colDec1=None,
         if max_distance > 180 * u.arcsec:
             raise ValueError(
                 'max_distance argument must not be greater than 180')
-        payload = {
-            'request': 'xmatch',
-            'distMaxArcsec': max_distance.to(u.arcsec).value,
-            'RESPONSEFORMAT': 'csv',
-            **kwargs
-        }
+        payload = {'request': 'xmatch',
+                   'distMaxArcsec': max_distance.to(u.arcsec).value,
+                   'RESPONSEFORMAT': 'votable',
+                   **kwargs}
+
         kwargs = {}
 
         self._prepare_sending_table(1, payload, kwargs, cat1, colRA1, colDec1)
@@ -190,22 +190,5 @@ def get_available_tables(self, cache=True):
         content = response.text
         return content.splitlines()
 
-    def _parse_text(self, text):
-        """
-        Parse a CSV text file that has potentially duplicated header names
-        """
-        header = text.split("\n")[0]
-        colnames = header.split(",")
-        for column in colnames:
-            if colnames.count(column) > 1:
-                counter = 1
-                while colnames.count(column) > 0:
-                    colnames[colnames.index(column)] = column + "_{counter}".format(counter=counter)
-                    counter += 1
-        new_text = ",".join(colnames) + "\n" + "\n".join(text.split("\n")[1:])
-        result = ascii.read(new_text, format='csv', fast_reader=False)
-
-        return result
-
 
 XMatch = XMatchClass()

astroquery/xmatch/tests/test_xmatch.py

@@ -104,9 +104,3 @@ def test_xmatch_query_cat1_table_local(monkeypatch):
         'errHalfMaj', 'errHalfMin', 'errPosAng', 'Jmag', 'Hmag', 'Kmag',
         'e_Jmag', 'e_Hmag', 'e_Kmag', 'Qfl', 'Rfl', 'X', 'MeasureJD']
     assert len(table) == 11
-
-
-@pytest.mark.parametrize('datafile', DATA_FILES.values())
-def test_parse_text(datafile):
-    xm = XMatch()
-    xm._parse_text(datafile)

astroquery/xmatch/tests/test_xmatch_remote.py

@@ -5,10 +5,10 @@
 import pytest
 import requests
 from requests import ReadTimeout
+from numpy.testing import assert_allclose
 
 from astropy.table import Table
 from astropy.units import arcsec, arcmin
-from astropy.io import ascii
 
 from astropy.coordinates import SkyCoord
 
@@ -37,12 +37,15 @@ def test_is_xmatch_up():
 def remote_table(tmp_path_factory):
     # this can be used to check that the API is still functional & doing as expected
     infile = DATA_DIR / "posList.csv"
-    outfile = tmp_path_factory.mktemp("remote_data") / "http_result.csv"
-    os.system("curl -X POST -F request=xmatch -F distMaxArcsec=5 -F RESPONSEFORMAT=csv "
+    outfile = tmp_path_factory.mktemp("remote_data") / "http_result.vot"
+    os.system("curl -X POST -F request=xmatch -F distMaxArcsec=5 -F RESPONSEFORMAT=votable "
               "-F cat1=@{1} -F colRA1=ra -F colDec1=dec -F cat2=vizier:II/246/out  "
               "http://cdsxmatch.u-strasbg.fr/xmatch/api/v1/sync > {0}".
               format(outfile, infile))
-    return ascii.read(outfile, format="csv", fast_reader=False)
+    remote_table = Table.read(outfile, format="votable")
+    remote_table.rename_column('_2MASS', '2MASS')
+
+    return remote_table
 
 
 @pytest.mark.remote_data
@@ -83,7 +86,11 @@ def test_xmatch_query(self, xmatch, remote_table):
             'e_Jmag', 'e_Hmag', 'e_Kmag', 'Qfl', 'Rfl', 'X', 'MeasureJD']
         assert len(table) == 11
 
-        assert all(table == remote_table)
+        for col in table.colnames:
+            if remote_table[col].dtype.kind == 'U':
+                assert all(table[col] == remote_table[col])
+            else:
+                assert_allclose(table[col], remote_table[col])
 
     def test_xmatch_query_astropy_table(self, xmatch, remote_table):
         input_table = Table.read(DATA_DIR / "posList.csv", format="ascii.csv")
@@ -99,8 +106,11 @@ def test_xmatch_query_astropy_table(self, xmatch, remote_table):
             'errHalfMaj', 'errHalfMin', 'errPosAng', 'Jmag', 'Hmag', 'Kmag',
             'e_Jmag', 'e_Hmag', 'e_Kmag', 'Qfl', 'Rfl', 'X', 'MeasureJD']
         assert len(table) == 11
-
-        assert all(table == remote_table)
+        for col in table.colnames:
+            if remote_table[col].dtype.kind == 'U':
+                assert all(table[col] == remote_table[col])
+            else:
+                assert_allclose(table[col], remote_table[col])
 
     @pytest.mark.skipif('regions' not in sys.modules,
                         reason="requires astropy-regions")

docs/xmatch/xmatch.rst

@@ -55,6 +55,7 @@ in the resulting table for demonstration purposes.  Finally, ``colRa1`` and
     <class 'astropy.table.table.Table'>
     >>> print(table)
     angDist      ra       dec         2MASS       ... Qfl Rfl  X   MeasureJD
+     arcsec                                       ...                  d
     -------- --------- --------- ---------------- ... --- --- --- ------------
     1.352044 267.22029 -20.35869 17485281-2021323 ... EEU 226   2 2450950.8609
     1.578188 267.22029 -20.35869 17485288-2021328 ... UUB 662   2 2450950.8609