Merge pull request #1980 from ceb8/mast_anonymous_aws

MAST: making AWS clous access anonymous

Author: C. E. Brasseur

CHANGES.rst

@@ -43,6 +43,7 @@ mast
 - Added Zcut functionality to astroquery [#1911]
 - Fixed error causing empty products passed to ``Observations.get_product_list()`` to yeild a
   non-empty result. [#1921]
+- Changed AWS cloud access from RequesterPays to anonymous acces [#1980]
 
 esa/hubble
 ^^^^^^^^^^

astroquery/mast/cloud.py

@@ -13,6 +13,7 @@
 
 from astropy.logger import log
 from astropy.utils.console import ProgressBarOrSpinner
+from astropy.utils.exceptions import AstropyDeprecationWarning
 
 from ..exceptions import NoResultsWarning, InvalidQueryError
 
@@ -44,25 +45,24 @@ def __init__(self, provider="AWS", profile=None, verbose=False):
             Default False. Display extra info and warnings if true.
         """
 
+        # Dealing with deprecated argument
+        if profile is not None:
+            warnings.warn(("MAST Open Data on AWS is now free to access and does "
+                           "not require an AWS account"), AstropyDeprecationWarning)
+
         import boto3
         import botocore
 
         self.supported_missions = ["mast:hst/product", "mast:tess/product", "mast:kepler"]
 
-        if profile is not None:
-            self.boto3 = boto3.Session(profile_name=profile)
-        else:
-            self.boto3 = boto3
+        self.boto3 = boto3
         self.botocore = botocore
+        self.config = botocore.client.Config(signature_version=botocore.UNSIGNED)
 
         self.pubdata_bucket = "stpubdata"
 
         if verbose:
             log.info("Using the S3 STScI public dataset")
-            log.warning("Your AWS account will be charged for access to the S3 bucket")
-            log.info("See Request Pricing in https://aws.amazon.com/s3/pricing/ for details")
-            log.info("If you have not configured boto3, follow the instructions here: "
-                     "https://boto3.readthedocs.io/en/latest/guide/configuration.html")
 
     def is_supported(self, data_product):
         """
@@ -110,7 +110,7 @@ def get_cloud_uri(self, data_product, include_bucket=True, full_url=False):
             found in the cloud, None is returned.
         """
 
-        s3_client = self.boto3.client('s3')
+        s3_client = self.boto3.client('s3', config=self.config)
 
         path = utils.mast_relative_path(data_product["dataURI"])
         if path is None:
@@ -119,7 +119,7 @@ def get_cloud_uri(self, data_product, include_bucket=True, full_url=False):
         path = path.lstrip("/")
 
         try:
-            s3_client.head_object(Bucket=self.pubdata_bucket, Key=path, RequestPayer='requester')
+            s3_client.head_object(Bucket=self.pubdata_bucket, Key=path)
             if include_bucket:
                 path = "s3://{}/{}".format(self.pubdata_bucket, path)
             elif full_url:
@@ -172,8 +172,8 @@ def download_file(self, data_product, local_path, cache=True):
             Default is True. If file is found on disc it will not be downloaded again.
         """
 
-        s3 = self.boto3.resource('s3')
-        s3_client = self.boto3.client('s3')
+        s3 = self.boto3.resource('s3', config=self.config)
+        s3_client = self.boto3.client('s3', config=self.config)
         bkt = s3.Bucket(self.pubdata_bucket)
         with warnings.catch_warnings():
             warnings.simplefilter("ignore")
@@ -182,7 +182,7 @@ def download_file(self, data_product, local_path, cache=True):
             raise Exception("Unable to locate file {}.".format(data_product['productFilename']))
 
         # Ask the webserver (in this case S3) what the expected content length is and use that.
-        info_lookup = s3_client.head_object(Bucket=self.pubdata_bucket, Key=bucket_path, RequestPayer='requester')
+        info_lookup = s3_client.head_object(Bucket=self.pubdata_bucket, Key=bucket_path)
         length = info_lookup["ContentLength"]
 
         if cache and os.path.exists(local_path):
@@ -219,5 +219,4 @@ def progress_callback(numbytes):
                     bytes_read += numbytes
                     pb.update(bytes_read)
 
-            bkt.download_file(bucket_path, local_path, ExtraArgs={"RequestPayer": "requester"},
-                              Callback=progress_callback)
+            bkt.download_file(bucket_path, local_path, Callback=progress_callback)

astroquery/mast/observations.py

@@ -539,7 +539,7 @@ def download_file(self, uri, local_path=None, base_url=None, cache=True, cloud_o
             local_path = os.path.join(os.path.abspath('.'), filename)
 
         # recreate the data_product key for cloud connection check
-        data_product = {'dataURI': data_url}
+        data_product = {'dataURI': uri}
 
         status = "COMPLETE"
         msg = None