Merge pull request #601 from keflavich/alma-private-fix

BUGFIX:  ALMA private queries, plus checking that queries succeed

Author: keflavich

astroquery/alma/core.py

@@ -293,11 +293,20 @@ def stage_data(self, uids):
         summary.raise_for_status()
         self._staging_log['json_data'] = json_data = summary.json()
 
+        username = self._username if hasattr(self, '_username') else 'anonymous'
+
+        # templates:
+        # https://almascience.eso.org/dataPortal/requests/keflavich/946895898/ALMA/
+        # 2013.1.00308.S_uid___A001_X196_X93_001_of_001.tar/2013.1.00308.S_uid___A001_X196_X93_001_of_001.tar
+        # uid___A002_X9ee74a_X26f0/2013.1.00308.S_uid___A002_X9ee74a_X26f0.asdm.sdm.tar
+
         url_decomposed = urlparse(data_page_url)
         base_url = ('{uri.scheme}://{uri.netloc}/'
-                    'dataPortal/requests/anonymous/'
+                    'dataPortal/requests/{username}/'
                     '{staging_page_id}/ALMA'.format(uri=url_decomposed,
-                                                    staging_page_id=dpid))
+                                                    staging_page_id=dpid,
+                                                    username=username,
+                                                   ))
         tbl = self._json_summary_to_table(json_data, base_url=base_url)
 
         # staging_root = BeautifulSoup(data_page.content)
@@ -353,7 +362,7 @@ def download_files(self, files, cache=True):
         downloaded_files = []
         for fileLink in unique(files):
             filename = self._request("GET", fileLink, save=True,
-                                     timeout=self.TIMEOUT)
+                                     timeout=self.TIMEOUT, cache=cache)
             downloaded_files.append(filename)
         return downloaded_files
 
@@ -444,6 +453,7 @@ def _login(self, username, store_password=False):
 
         if authenticated:
             log.info("Authentication successful!")
+            self._username = username
         else:
             log.exception("Authentication failed!")
         # When authenticated, save password in keyring if needed
@@ -882,18 +892,27 @@ def _json_summary_to_table(self, data, base_url):
         """
         columns = {'uid':[], 'URL':[], 'size':[]}
         for entry in data['node_data']:
-            if entry['file_name'] != 'null':
+            is_file = (entry['de_type'] == 'MOUS'
+                       or (entry['file_name'] != 'null'
+                           and entry['file_key'] != 'null'))
+            if is_file:
                 # "de_name": "ALMA+uid://A001/X122/X35e",
                 columns['uid'].append(entry['de_name'][5:])
                 columns['size'].append((int(entry['file_size'])*u.B).to(u.Gbyte))
                 # example template for constructing url:
                 # https://almascience.eso.org/dataPortal/requests/keflavich/940238268/ALMA/
                 # uid___A002_X9d6f4c_X154/2013.1.00546.S_uid___A002_X9d6f4c_X154.asdm.sdm.tar
-                # above is WRONG
+                # above is WRONG... except for ASDMs, when it's right
                 # should be:
                 # 2013.1.00546.S_uid___A002_X9d6f4c_X154.asdm.sdm.tar/2013.1.00546.S_uid___A002_X9d6f4c_X154.asdm.sdm.tar
+                #
+                # apparently ASDMs are different from others:
+                # templates:
+                # https://almascience.eso.org/dataPortal/requests/keflavich/946895898/ALMA/
+                # 2013.1.00308.S_uid___A001_X196_X93_001_of_001.tar/2013.1.00308.S_uid___A001_X196_X93_001_of_001.tar
+                # uid___A002_X9ee74a_X26f0/2013.1.00308.S_uid___A002_X9ee74a_X26f0.asdm.sdm.tar
                 url = os.path.join(base_url,
-                                   entry['file_name'],
+                                   entry['file_key'],
                                    entry['file_name'],
                                   )
                 columns['URL'].append(url)

astroquery/alma/tests/test_alma_remote.py

@@ -70,7 +70,9 @@ def test_stage_data(self, temp_dir):
         assert b'2011.0.00217.S' in result_s['Project code']
         uid = result_s['Asdm uid'][0]
 
-        alma.stage_data([uid])
+        result = alma.stage_data([uid])
+
+        assert os.path.split(result['URL'][0])[1] == 'uid___A002_X47ed8e_X6c.asdm.sdm.tar'
 
     def test_doc_example(self, temp_dir):
         alma = Alma()

astroquery/query.py

@@ -152,9 +152,9 @@ def _request(self, method, url, params=None, data=None, headers=None,
             local_filename = url.split('/')[-1]
             local_filepath = os.path.join(self.cache_location or savedir or
                                           '.', local_filename)
-            log.info("Downloading {0}...".format(local_filename))
+            # REDUNDANT: spinner has this log.info("Downloading {0}...".format(local_filename))
             self._download_file(url, local_filepath, timeout=timeout,
-                                auth=auth)
+                                auth=auth, cache=cache)
             return local_filepath
         else:
             query = AstroQuery(method, url, params=params, data=data,
@@ -174,24 +174,45 @@ def _request(self, method, url, params=None, data=None, headers=None,
                     to_cache(response, query.request_file(self.cache_location))
             return response
 
-    def _download_file(self, url, local_filepath, timeout=None, auth=None):
+    def _download_file(self, url, local_filepath, timeout=None, auth=None, cache=False):
         """
         Download a file.  Resembles `astropy.utils.data.download_file` but uses
         the local ``_session``
         """
         response = self._session.get(url, timeout=timeout, stream=True,
-                                      auth=auth)
+                                     auth=auth)
+        response.raise_for_status()
         if 'content-length' in response.headers:
             length = int(response.headers['content-length'])
         else:
             length = None
 
+        if cache and os.path.exists(local_filepath):
+            if length is not None:
+                statinfo = os.stat(local_filepath)
+                if statinfo.st_size != length:
+                    log.warn("Found cached file {0} with size {1} that is "
+                             "different from expected size {2}"
+                             .format(local_filepath,
+                                     statinfo.st_size,
+                                     length))
+                else:
+                    log.info("Found cached file {0} with expected size {1}."
+                             .format(local_filepath, statinfo.st_size))
+                    response.close()
+                    return
+            else:
+                log.info("Found cached file {0}.".format(local_filepath))
+                response.close()
+                return
+
         blocksize = astropy.utils.data.conf.download_block_size
 
         bytes_read = 0
 
         with ProgressBarOrSpinner(length,
-                                  'Downloading URL {0} ...'.format(url)) as pb:
+                                  'Downloading URL {0} to {1} ...'.format(url,
+                                                                          local_filepath)) as pb:
             with open(local_filepath, 'wb') as f:
                 for block in response.iter_content(blocksize):
                     f.write(block)