Merge pull request #2158 from jd-au/CASDA-6719-stage-data

Allow non image data to be retrieved

Author: bsipocz

CHANGES.rst

@@ -10,6 +10,12 @@ New Tools and Services
 Service fixes and enhancements
 ------------------------------
 
+casda
+^^^^^^
+
+- Add ability to stage and download non image data which have been found 
+  through the CASDA obscore table. [#2158]
+
 vizier
 ^^^^^^
 

README.rst

@@ -109,6 +109,7 @@ The following modules have been completed using a common API:
   * `Besancon <http://astroquery.readthedocs.io/en/latest/besancon/besancon.html>`_: Model of stellar population synthesis in the Galaxy.
   * `CDS MOC Service <https://astroquery.readthedocs.io/en/latest/cds/cds.html>`_: A collection of all-sky survey coverage maps.
   * `CADC <https://astroquery.readthedocs.io/en/latest/cadc/cadc.html>`_: Canadian Astronomy Data Centre.
+  * `CASDA <https://astroquery.readthedocs.io/en/latest/casda/casda.html>`_:  CSIRO ASKAP Science Data Archive.
   * `ESASky <http://astroquery.readthedocs.io/en/latest/esasky/esasky.html>`_: ESASky is a science driven discovery portal providing easy visualizations and full access to the entire sky as observed with ESA Space astronomy missions.
   * `ESO Archive <http://astroquery.readthedocs.io/en/latest/eso/eso.html>`_
   * `FIRST <http://astroquery.readthedocs.io/en/latest/image_cutouts/first/first.html>`_: Faint Images of the Radio Sky at Twenty-cm. 20-cm radio images of the extragalactic sky from the VLA.

astroquery/casda/core.py

@@ -184,13 +184,21 @@ def stage_data(self, table, verbose=False):
 
         # Use datalink to get authenticated access for each file
         tokens = []
+        soda_url = None
         for row in table:
             access_url = row['access_url']
-            response = self._request('GET', access_url, auth=self._auth,
-                                     timeout=self.TIMEOUT, cache=False)
-            response.raise_for_status()
-            soda_url, id_token = self._parse_datalink_for_service_and_id(response, 'cutout_service')
-            tokens.append(id_token)
+            if access_url:
+                response = self._request('GET', access_url, auth=self._auth,
+                                        timeout=self.TIMEOUT, cache=False)
+                response.raise_for_status()
+                service_url, id_token = self._parse_datalink_for_service_and_id(response, 'async_service')
+                if id_token:
+                    tokens.append(id_token)
+                    soda_url = service_url
+
+        # Trap a request with no allowed data
+        if not soda_url:
+            raise ValueError('You do not have access to any of the requested data files.')
 
         # Create job to stage all files
         job_url = self._create_soda_job(tokens, soda_url=soda_url)

astroquery/casda/tests/data/datalink_noaccess.xml

@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?><?xml-stylesheet href='https://data.csiro.au/casda_vo_proxy/vo/votable.xsl' type='text/xsl' ?>
+
+<VOTABLE xmlns="http://www.ivoa.net/xml/VOTable/v1.3" version="1.3">
+    <RESOURCE name="CASDA Datalink Result" type="results">
+        <TABLE>
+            <FIELD datatype="char" name="ID" ucd="meta.id;meta.main" arraysize="*"/>
+            <FIELD datatype="char" name="access_url" ucd="meta.ref.url" arraysize="*"/>
+            <FIELD datatype="char" name="service_def" ucd="meta.ref" arraysize="*"/>
+            <FIELD datatype="char" name="error_message" ucd="meta.code.error" arraysize="*"/>
+            <FIELD datatype="char" name="description" ucd="meta.note" arraysize="*"/>
+            <FIELD datatype="char" name="semantics" ucd="meta.code" arraysize="*"/>
+            <FIELD datatype="char" name="content_type" ucd="meta.code.mime" arraysize="*"/>
+            <FIELD unit="byte" datatype="long" name="content_length" ucd="phys.size;meta.file"/>
+            <FIELD ID="authenticatedIdToken" datatype="char" name="authenticated_id_token" ucd="meta.id" arraysize="*"/>
+            <DATA>
+                <TABLEDATA>
+                    <TR>
+                        <TD>3242</TD>
+                        <TD/>
+                        <TD/>
+                        <TD>DefaultFault: Sorry, but you do not have permission to access this data product</TD>
+                        <TD/>
+                        <TD>#error</TD>
+                        <TD/>
+                        <TD/>
+                        <TD/>
+                    </TR>
+                </TABLEDATA>
+            </DATA>
+        </TABLE>
+    </RESOURCE>
+</VOTABLE>

astroquery/casda/tests/test_casda.py

@@ -21,7 +21,7 @@
     pytest.skip("Install mock for the casda tests.", allow_module_level=True)
 
 DATA_FILES = {'CIRCLE': 'cone.xml', 'RANGE': 'box.xml', 'DATALINK': 'datalink.xml', 'RUN_JOB': 'run_job.xml',
-              'COMPLETED_JOB': 'completed_job.xml'}
+              'COMPLETED_JOB': 'completed_job.xml', 'DATALINK_NOACCESS': 'datalink_noaccess.xml'}
 
 
 class MockResponse:
@@ -59,7 +59,10 @@ def get_mockreturn(self, method, url, data=None, timeout=10,
         else:
             raise ValueError("Unexpected SODA async {} call to url {}".format(method, url))
     elif 'datalink' in str(url):
-        key = 'DATALINK'
+        if 'cube-244' in str(url):
+            key = 'DATALINK'
+        else:
+            key = 'DATALINK_NOACCESS'
     else:
         key = params['POS'].split()[0] if params['POS'] else None
     filename = data_path(DATA_FILES[key])
@@ -257,6 +260,19 @@ def test_stage_data_invalid_credentials(patch_get):
         casda.stage_data(table)
 
 
+def test_stage_data_no_link(patch_get):
+    prefix = 'https://somewhere/casda/datalink/links?'
+    access_urls = [prefix + 'cube-240']
+    table = Table([Column(data=access_urls, name='access_url')])
+    casda = Casda('user', 'password')
+    casda.POLL_INTERVAL = 1
+
+    with pytest.raises(ValueError) as excinfo:
+        casda.stage_data(table)
+
+    assert "You do not have access to any of the requested data files." in str(excinfo.value)
+
+
 def test_stage_data(patch_get):
     prefix = 'https://somewhere/casda/datalink/links?'
     access_urls = [prefix + 'cube-244']

docs/casda/casda.rst

@@ -27,8 +27,8 @@ For example:
     >>> from astropy import units as u
     >>> centre = SkyCoord.from_name('NGC 7232')
     >>> result_table = Casda.query_region(centre, radius=30*u.arcmin)
-    >>> print(result_table['obs_publisher_did','s_ra', 's_dec', 'obs_released_date'])
-    obs_publisher_did       s_ra           s_dec          obs_released_date
+    >>> print(result_table['obs_publisher_did','s_ra', 's_dec', 'obs_release_date'])
+    obs_publisher_did       s_ra           s_dec          obs_release_date
                             deg             deg
     ----------------- --------------- ---------------- ------------------------
              cube-502 333.16767306594 -45.302084636451 2017-08-02T03:51:19.728Z
@@ -64,8 +64,8 @@ For example to filter out the 30 non-public results from the above data set:
 .. code-block:: python
 
     >>> public_results = Casda.filter_out_unreleased(result_table)
-    >>> print(public_results['obs_publisher_did','s_ra', 's_dec', 'obs_released_date'])
-    obs_publisher_did       s_ra           s_dec          obs_released_date
+    >>> print(public_results['obs_publisher_did','s_ra', 's_dec', 'obs_release_date'])
+    obs_publisher_did       s_ra           s_dec          obs_release_date
                             deg             deg
     ----------------- --------------- ---------------- ------------------------
              cube-502 333.16767306594 -45.302084636451 2017-08-02T03:51:19.728Z