provide profile as part of S3ClientConfig (#341)

provide profile as part of S3ClientConfig

---------

Co-authored-by: Tal Ifargan <[email protected]>

Author: TalIfargan

README.md

@@ -41,6 +41,7 @@ that role.
 - Set credentials in the AWS credentials profile file on the local system, located at: `~/.aws/credentials` 
 on Unix or macOS.
 - Set the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables.
+- Pass the name of the desired profile that you have configured in `~/.aws/config` and `~/.aws/credentials` to the `S3ClientConfig` object.
 
 ### Examples
 

s3torchconnector/src/s3torchconnector/_s3client/_s3client.py

@@ -136,6 +136,7 @@ def _client_builder(self) -> MountpointS3Client:
         return MountpointS3Client(
             region=self._region,
             endpoint=self._endpoint,
+            profile=self._s3client_config.profile,
             user_agent_prefix=self._user_agent_prefix,
             throughput_target_gbps=self._s3client_config.throughput_target_gbps,
             part_size=self._s3client_config.part_size,

s3torchconnector/src/s3torchconnector/_s3client/s3client_config.py

@@ -1,6 +1,7 @@
 #  Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #  // SPDX-License-Identifier: BSD
 from dataclasses import dataclass
+from typing import Optional
 
 
 @dataclass(frozen=True)
@@ -15,12 +16,15 @@ class S3ClientConfig:
         (max number of parts per upload is 10,000, minimum upload part size is 5 MiB).
         Part size must have values between 5MiB and 5GiB.
         8MiB by default (may change in future).
+    unsigned(bool): Set to true to disable signing S3 requests.
     force_path_style(bool): forceful path style addressing for S3 client.
-    max_attempts(int): amount of retry attempts for retrieable errors
+    max_attempts(int): amount of retry attempts for retrieable errors.
+    profile(str): Profile name to use for S3 authentication.
     """
 
     throughput_target_gbps: float = 10.0
     part_size: int = 8 * 1024 * 1024
     unsigned: bool = False
     force_path_style: bool = False
     max_attempts: int = 10
+    profile: Optional[str] = None

s3torchconnector/tst/e2e/test_s3_client.py

@@ -1,12 +1,15 @@
 #  Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #  // SPDX-License-Identifier: BSD
 
+import os
+import tempfile
 import pytest
 from s3torchconnectorclient import S3Exception
 
-from s3torchconnector._s3client import S3Client
+from s3torchconnector._s3client import S3Client, S3ClientConfig
 
 HELLO_WORLD_DATA = b"Hello, World!\n"
+TEST_PROFILE_NAME = "test-profile"
 
 
 def test_no_access_objects_without_profile(empty_directory):
@@ -24,3 +27,46 @@ def test_no_access_objects_without_profile(empty_directory):
             filename,
         )
         put_stream.write(HELLO_WORLD_DATA)
+
+
+def test_access_objects_with_profile(empty_directory):
+    if empty_directory.profile_bucket is None:
+        pytest.skip("No profile bucket configured")
+
+    try:
+        tmp_file = tempfile.NamedTemporaryFile()
+        tmp_file.write(
+            f"""[profile default]
+aws_access_key_id = {os.getenv("AWS_ACCESS_KEY_ID")}
+aws_secret_access_key = {os.getenv("AWS_SECRET_ACCESS_KEY")}
+aws_session_token = {os.getenv("AWS_SESSION_TOKEN")}
+ 
+[profile {TEST_PROFILE_NAME}]
+role_arn = {empty_directory.profile_arn}
+region = {empty_directory.region}
+source_profile = default""".encode()
+        )
+        tmp_file.flush()
+        os.environ["AWS_CONFIG_FILE"] = tmp_file.name
+
+        client = S3Client(
+            empty_directory.region,
+            s3client_config=S3ClientConfig(profile=TEST_PROFILE_NAME),
+        )
+        filename = f"{empty_directory.prefix}hello_world.txt"
+
+        put_stream = client.put_object(
+            empty_directory.profile_bucket,
+            filename,
+        )
+
+        put_stream.write(HELLO_WORLD_DATA)
+        put_stream.close()
+
+        get_stream = client.get_object(
+            empty_directory.profile_bucket,
+            filename,
+        )
+        assert b"".join(get_stream) == HELLO_WORLD_DATA
+    finally:
+        os.environ["AWS_CONFIG_FILE"] = ""

s3torchconnector/tst/unit/test_s3_client_config.py

@@ -13,13 +13,19 @@ def test_default():
     assert config.throughput_target_gbps == 10.0
     assert config.force_path_style is False
     assert config.max_attempts == 10
+    assert config.profile is None
 
 
 def test_enable_force_path_style():
     config = S3ClientConfig(force_path_style=True)
     assert config.force_path_style is True
 
 
+def test_change_profile():
+    config = S3ClientConfig(profile="test_profile")
+    assert config.profile == "test_profile"
+
+
 @given(part_size=integers(min_value=5 * MiB, max_value=5 * GiB))
 def test_part_size_setup(part_size: int):
     config = S3ClientConfig(part_size=part_size)