Add warning to use if duplicate downloads are culled

Author: jdavies-st

astroquery/exceptions.py

@@ -7,8 +7,9 @@
 
 __all__ = ['TimeoutError', 'InvalidQueryError', 'RemoteServiceError',
            'TableParseError', 'LoginError', 'ResolverError',
-           'NoResultsWarning', 'LargeQueryWarning', 'InputWarning',
-           'AuthenticationWarning', 'MaxResultsWarning', 'CorruptDataWarning']
+           'NoResultsWarning', 'DuplicateResultsWarning', 'LargeQueryWarning',
+           'InputWarning', 'AuthenticationWarning', 'MaxResultsWarning',
+           'CorruptDataWarning']
 
 
 class TimeoutError(Exception):
@@ -67,6 +68,13 @@ class NoResultsWarning(AstropyWarning):
     pass
 
 
+class DuplicateResultsWarning(AstropyWarning):
+    """
+    Astroquery warning class to be issued when a query returns no result.
+    """
+    pass
+
+
 class LargeQueryWarning(AstropyWarning):
     """
     Astroquery warning class to be issued when a query is larger than

astroquery/mast/observations.py

@@ -31,7 +31,7 @@
 from ..utils import commons, async_to_sync
 from ..utils.class_or_instance import class_or_instance
 from ..exceptions import (TimeoutError, InvalidQueryError, RemoteServiceError,
-                          ResolverError, MaxResultsWarning,
+                          ResolverError, MaxResultsWarning, DuplicateResultsWarning,
                           NoResultsWarning, InputWarning, AuthenticationWarning)
 
 from . import conf, utils
@@ -715,7 +715,12 @@ def download_products(self, products, *, download_dir=None,
             products = vstack(product_lists)
 
         # Remove duplicate products
+        number = len(products)
         products = unique(products, keys="dataURI")
+        number_unique = len(products)
+        if number_unique < number:
+            warnings.warn(f"{number - number_unique} of {number} products were duplicates."
+                          f"Only downloading {number_unique} unique product(s).", DuplicateResultsWarning)
 
         # apply filters
         products = self.filter_products(products, mrp_only=mrp_only, **filters)

astroquery/mast/tests/test_mast_remote.py

@@ -14,7 +14,8 @@
 from astroquery import mast
 
 from ..utils import ResolverError
-from ...exceptions import InvalidQueryError, MaxResultsWarning, NoResultsWarning, RemoteServiceError
+from ...exceptions import (InvalidQueryError, MaxResultsWarning, NoResultsWarning,
+                           DuplicateResultsWarning, RemoteServiceError)
 
 
 OBSID = '1647157'
@@ -302,8 +303,9 @@ def test_observations_download_products_noduplicates(self, tmpdir):
         assert len(products) == 6
 
         # Download the product
-        manifest = mast.Observations.download_products(products,
-                                                       download_dir=str(tmpdir))
+        with pytest.warns(DuplicateResultsWarning):
+            manifest = mast.Observations.download_products(products,
+                                                           download_dir=str(tmpdir))
 
         # Check that it downloads the MSA config file only once
         assert len(manifest) == 1