1128 s3 proxy support with data wrangler (#1361)

* exposing 'verify' argument for boto3 session classes

* formatting

* adding 'verify' to config

* Modify tutorial to include verify and test again

* Tmp fix - Reduce size test shot

* Increase size test

* Run full test

* Run full tests without rerun

* Fix _config to not delete verify

Co-authored-by: lucas hanson <[email protected]>
Co-authored-by: Abdel Jaidi <[email protected]>
Co-authored-by: kukushking <[email protected]>

Author: 4 people

awswrangler/_config.py

@@ -48,6 +48,7 @@ class _ConfigArg(NamedTuple):
     "secretsmanager_endpoint_url": _ConfigArg(dtype=str, nullable=True, enforced=True),
     # Botocore config
     "botocore_config": _ConfigArg(dtype=botocore.config.Config, nullable=True),
+    "verify": _ConfigArg(dtype=str, nullable=True),
 }
 
 
@@ -68,6 +69,7 @@ def __init__(self) -> None:
         self.dynamodb_endpoint_url = None
         self.secretsmanager_endpoint_url = None
         self.botocore_config = None
+        self.verify = None
         for name in _CONFIG_ARGS:
             self._load_config(name=name)
 
@@ -153,7 +155,7 @@ def __getitem__(self, item: str) -> _ConfigValueType:
 
     def _reset_item(self, item: str) -> None:
         if item in self._loaded_values:
-            if item.endswith("_endpoint_url"):
+            if item.endswith("_endpoint_url") or item == "verify":
                 self._loaded_values[item] = None
             else:
                 del self._loaded_values[item]
@@ -394,6 +396,15 @@ def botocore_config(self) -> botocore.config.Config:
     def botocore_config(self, value: Optional[botocore.config.Config]) -> None:
         self._set_config_value(key="botocore_config", value=value)
 
+    @property
+    def verify(self) -> Optional[str]:
+        """Property verify."""
+        return cast(Optional[str], self["verify"])
+
+    @verify.setter
+    def verify(self, value: Optional[str]) -> None:
+        self._set_config_value(key="verify", value=value)
+
 
 def _insert_str(text: str, token: str, insert: str) -> str:
     """Insert string into other."""

awswrangler/_utils.py

@@ -115,8 +115,8 @@ def client(
         service_name=service_name,
         endpoint_url=endpoint_url,
         use_ssl=True,
-        verify=verify,
-        config=default_botocore_config() if botocore_config is None else botocore_config,
+        config=botocore_config or default_botocore_config(),
+        verify=verify or _config.config.verify,
     )