Merge pull request #3290 from snbianco/content-length-fix

Log a warning if a cached file's length cannot be verified

Author: bsipocz

astroquery/mast/tests/test_mast_remote.py

@@ -1,5 +1,6 @@
 # Licensed under a 3-clause BSD style license - see LICENSE.rst
 
+import logging
 from pathlib import Path
 import numpy as np
 import os
@@ -769,6 +770,23 @@ def test_observations_download_file_escaped(self, tmp_path):
         f = fits.open(Path(tmp_path, filename))
         f.close()
 
+    def test_observations_download_file_no_length(self, tmp_path, caplog):
+        # test that `download_file` correctly handles the case where the server
+        # does not return a Content-Length header for a cached file
+        # initial download
+        in_uri = "mast:HLA/url/cgi-bin/getdata.cgi?filename=hst_05206_01_wfpc2_f375n_wf_daophot_trm.cat"
+        filename = Path(in_uri).name
+        result = Observations.download_file(uri=in_uri, local_path=tmp_path)
+        assert result == ("COMPLETE", None, None)
+        assert Path(tmp_path, filename).exists()
+
+        # download again, should warn and re-download file
+        with caplog.at_level(logging.WARNING):
+            result = Observations.download_file(uri=in_uri, local_path=tmp_path)
+        assert "Could not verify length of cached file" in caplog.text
+        assert result == ("COMPLETE", None, None)
+        assert Path(tmp_path, filename).exists()
+
     @pytest.mark.parametrize("test_data_uri, expected_cloud_uri", [
         ("mast:HST/product/u24r0102t_c1f.fits",
          "s3://stpubdata/hst/public/u24r/u24r0102t/u24r0102t_c1f.fits"),

astroquery/query.py

@@ -487,9 +487,17 @@ def _download_file(self, url, local_filepath, timeout=None, auth=None,
                     response.close()
                     return local_filepath
             else:
-                # This case doesn't appear reachable under normal circumstances
-                # It is not covered by tests, and probably indicates a badly-behaved server
-                raise ValueError(f"Found cached file {local_filepath}.  Could not verify length.")
+                # This is a special case where the server doesn't return a
+                # Content-Length header, but the file is already cached.
+                # One such case is dynamically generated files in the MAST Archive.
+                # In this case, we warn the user and re-download the file.
+                log.warning(f"Could not verify length of cached file {local_filepath}. "
+                            "Re-downloading the file.")
+                open_mode = 'wb'
+                response = self._session.request(method, url,
+                                                 timeout=timeout, stream=True,
+                                                 auth=auth, **kwargs)
+                response.raise_for_status()
         else:
             open_mode = 'wb'
             if head_safe: