Merge pull request #2947 from esdc-esac-esa-int/ESA_gaia_fix_launch_job_for_json_GAIAPCR-1308

Gaia: Fix the astroquery method launch_job to retrieve the Table from the job when a json format is used

Author: bsipocz

CHANGES.rst

@@ -49,6 +49,9 @@ gaia
 
 - The method ``get_datalinks`` can be used with the new parameter linking_parameter. It completes PR #2859. [#2936]
 
+- Fix the exception thrown when the functions launch_job and launch_job_async retrieve the data for the json output_format but
+  do not dump the results to a file . [#2947]
+
 
 hsa
 ^^^

astroquery/gaia/tests/test_gaia_remote.py

@@ -8,51 +8,51 @@
 @pytest.mark.remote_data
 def test_query_object_columns_with_radius():
     # Regression test: `columns` were ignored if `radius` was provided [#2025]
-    Gaia = GaiaClass()
-    sc = SkyCoord(ra=0*u.deg, dec=0*u.deg)
-    table = Gaia.query_object_async(sc, radius=10*u.arcsec, columns=['ra'])
+    gaia = GaiaClass()
+    sc = SkyCoord(ra=0 * u.deg, dec=0 * u.deg)
+    table = gaia.query_object_async(sc, radius=10 * u.arcsec, columns=['ra'])
     assert table.colnames == ['ra', 'dist']
 
 
 @pytest.mark.remote_data
 def test_query_object_row_limit():
-    Gaia = GaiaClass()
+    gaia = GaiaClass()
     coord = SkyCoord(ra=280, dec=-60, unit=(u.degree, u.degree), frame='icrs')
     width = u.Quantity(0.1, u.deg)
     height = u.Quantity(0.1, u.deg)
-    r = Gaia.query_object_async(coordinate=coord, width=width, height=height)
+    r = gaia.query_object_async(coordinate=coord, width=width, height=height)
 
-    assert len(r) == Gaia.ROW_LIMIT
+    assert len(r) == gaia.ROW_LIMIT
 
-    Gaia.ROW_LIMIT = 10
-    r = Gaia.query_object_async(coordinate=coord, width=width, height=height)
+    gaia.ROW_LIMIT = 10
+    r = gaia.query_object_async(coordinate=coord, width=width, height=height)
 
-    assert len(r) == 10 == Gaia.ROW_LIMIT
+    assert len(r) == 10 == gaia.ROW_LIMIT
 
-    Gaia.ROW_LIMIT = -1
-    r = Gaia.query_object_async(coordinate=coord, width=width, height=height)
+    gaia.ROW_LIMIT = -1
+    r = gaia.query_object_async(coordinate=coord, width=width, height=height)
 
     assert len(r) == 184
 
 
 @pytest.mark.remote_data
 def test_cone_search_row_limit():
-    Gaia = GaiaClass()
+    gaia = GaiaClass()
     coord = SkyCoord(ra=280, dec=-60, unit=(u.degree, u.degree), frame='icrs')
     radius = u.Quantity(0.1, u.deg)
-    j = Gaia.cone_search_async(coord, radius=radius)
+    j = gaia.cone_search_async(coord, radius=radius)
     r = j.get_results()
 
-    assert len(r) == Gaia.ROW_LIMIT
+    assert len(r) == gaia.ROW_LIMIT
 
-    Gaia.ROW_LIMIT = 10
-    j = Gaia.cone_search_async(coord, radius=radius)
+    gaia.ROW_LIMIT = 10
+    j = gaia.cone_search_async(coord, radius=radius)
     r = j.get_results()
 
-    assert len(r) == 10 == Gaia.ROW_LIMIT
+    assert len(r) == 10 == gaia.ROW_LIMIT
 
-    Gaia.ROW_LIMIT = -1
-    j = Gaia.cone_search_async(coord, radius=radius)
+    gaia.ROW_LIMIT = -1
+    j = gaia.cone_search_async(coord, radius=radius)
     r = j.get_results()
 
     assert len(r) == 1218

astroquery/gaia/tests/test_gaiatap.py

@@ -362,7 +362,7 @@ def test_launch_job_async_json_format(tmp_path_factory, column_attrs_launch_json
         assert results[colname].dtype == attrs.dtype
 
 
-def test_launch_job_json_format(tmp_path_factory, column_attrs_launch_json, mock_querier_json, ):
+def test_launch_job_json_format(tmp_path_factory, column_attrs_launch_json, mock_querier_json):
     d = tmp_path_factory.mktemp("data") / 'launch_job.json'
     d.write_text(JOB_DATA_QUERIER_ASYNC_JSON, encoding="utf-8")
 
@@ -390,6 +390,29 @@ def test_launch_job_json_format(tmp_path_factory, column_attrs_launch_json, mock
         assert results[colname].dtype == attrs.dtype
 
 
+def test_launch_job_json_format_no_dump(tmp_path_factory, column_attrs_launch_json, mock_querier_json):
+    dump_to_file = False
+    output_format = 'json'
+    query = "SELECT TOP 1 source_id, ra, dec, parallax from gaiadr3.gaia_source"
+
+    job = mock_querier_json.launch_job(query, output_format=output_format, dump_to_file=dump_to_file)
+
+    assert job.async_ is False
+    assert job.get_phase() == "COMPLETED"
+    assert job.failed is False
+    # results
+    results = job.get_results()
+
+    assert type(results) is Table
+    assert 1 == len(results), len(results)
+
+    for colname, attrs in column_attrs_launch_json.items():
+        assert results[colname].name == attrs.name
+        assert results[colname].description == attrs.description
+        assert results[colname].unit == attrs.unit
+        assert results[colname].dtype == attrs.dtype
+
+
 def test_cone_search_and_changing_MAIN_GAIA_TABLE(mock_querier_async):
     # Regression test for #2093 and #2099 - changing the MAIN_GAIA_TABLE
     # had no effect.

astroquery/utils/tap/xmlparser/utils.py

@@ -16,9 +16,11 @@
 """
 import gzip
 import io
+import json
 
 from astropy import units as u
 from astropy.table import Table as APTable
+from astropy.table.table import Table
 
 
 def util_create_string_from_buffer(buffer):
@@ -35,8 +37,29 @@ def read_http_response(response, output_format, *, correct_units=True):
         result = APTable.read(io.BytesIO(gzip.decompress(data.read())), format=astropy_format)
     except OSError:
         # data is not a valid gzip file by BadGzipFile.
-        result = APTable.read(data, format=astropy_format)
-        pass
+
+        if output_format == 'json':
+
+            data = json.load(response)
+
+            if data.get('data') and data.get('metadata'):
+
+                column_name = []
+                for name in data['metadata']:
+                    column_name.append(name['name'])
+
+                result = Table(rows=data['data'], names=column_name, masked=True)
+
+                for v in data['metadata']:
+                    col_name = v['name']
+                    result[col_name].unit = v['unit']
+                    result[col_name].description = v['description']
+                    result[col_name].meta = {'metadata': v}
+
+            else:
+                result = APTable.read(data, format=astropy_format)
+        else:
+            result = APTable.read(data, format=astropy_format)
 
     if correct_units:
         modify_unrecognized_table_units(result)