tests are in progress

Author: keflavich

astroquery/alma/core.py

@@ -748,6 +748,23 @@ def download_files(self, files, savedir=None, cache=True,
                 filename = os.path.join(savedir,
                                         filename)
 
+            if verify_only:
+                existing_file_length = os.stat(filename).st_size
+                if 'content-length' in check_filename.headers:
+                    length = int(check_filename.headers['content-length'])
+                    if length == 0:
+                        warnings.warn('URL {0} has length=0'.format(url))
+                    elif existing_file_length == length:
+                        log.info(f"Found cached file {filename} with expected size {existing_file_length}.")
+                    elif existing_file_length < length:
+                        log.info(f"Found cached file {filename} with size {existing_file_length} < expected "
+                                 f"size {length}.  The download should be continued.")
+                    elif existing_file_length > length:
+                        warnings.warn(f"Found cached file {filename} with size {existing_file_length} > expected "
+                                      f"size {length}.  The download is likely corrupted.")
+                else:
+                    warnings.warn(f"Could not verify {url} because it has no 'content-length'")
+
             try:
                 if not verify_only:
                     self._download_file(file_link,

astroquery/alma/tests/test_alma_remote.py

@@ -3,6 +3,7 @@
 import shutil
 import numpy as np
 import pytest
+import warnings
 from datetime import datetime
 import os
 from urllib.parse import urlparse
@@ -655,3 +656,51 @@ def test_big_download_regression(alma):
 def test_download_html_file(alma):
     result = alma.download_files(['https://almascience.nao.ac.jp/dataPortal/member.uid___A001_X1284_X1353.qa2_report.html'])
     assert result
+
+
+@pytest.mark.remote_data
+def test_verify_html_file(alma, caplog):
+    # first, make sure the file is not cached (in case this test gets called repeatedly)
+    # (we are hacking the file later in this test to trigger different failure modes so
+    # we need it fresh)
+    try:
+        result = alma.download_files(['https://almascience.nao.ac.jp/dataPortal/member.uid___A001_X1284_X1353.qa2_report.html'], verify_only=True)
+        local_filepath = result[0]
+        os.remove(local_filepath)
+    except FileNotFoundError:
+        pass
+
+    caplog.clear()
+
+    # download the file
+    result = alma.download_files(['https://almascience.nao.ac.jp/dataPortal/member.uid___A001_X1284_X1353.qa2_report.html'])
+    assert result
+
+    result = alma.download_files(['https://almascience.nao.ac.jp/dataPortal/member.uid___A001_X1284_X1353.qa2_report.html'], verify_only=True)
+    assert result
+    local_filepath = result[0]
+    existing_file_length = 66336
+    assert f"Found cached file {local_filepath} with expected size {existing_file_length}." in caplog.text
+
+    # manipulate the file
+    with open(local_filepath, 'ab') as fh:
+        fh.write(b"Extra Text")
+
+    caplog.clear()
+    with warnings.catch_warnings() as ww:
+        result = alma.download_files(['https://almascience.nao.ac.jp/dataPortal/member.uid___A001_X1284_X1353.qa2_report.html'], verify_only=True)
+        assert result
+        length = 66336
+        existing_file_length = length + 10
+        assert f"Found cached file {local_filepath} with size {existing_file_length} > expected size {length}.  The download is likely corrupted." in str(ww)
+
+    # manipulate the file: make it small
+    with open(local_filepath, 'wb') as fh:
+        fh.write(b"Empty Text")
+
+    caplog.clear()
+    result = alma.download_files(['https://almascience.nao.ac.jp/dataPortal/member.uid___A001_X1284_X1353.qa2_report.html'], verify_only=True)
+    assert result
+    length = 66336
+    existing_file_length = 10
+    assert f"Found cached file {local_filepath} with size {existing_file_length} < expected size {length}.  The download should be continued." in caplog.text