Add endpoint variable for fixing CI (#147)

* Fix test workflow

Author: dnanuti

.github/workflows/python-integration.yml

@@ -30,10 +30,12 @@ jobs:
             bucket: ${{ vars.S3_BUCKET }}
             region: ${{ vars.S3_REGION }}
             storage-class: ""
+            endpoint-url: ${{ vars.S3_CUSTOM_ENDPOINT_URL }}
           - name: "S3 Express"
             bucket: ${{ vars.S3_EXPRESS_BUCKET }}
             region: ${{ vars.S3_EXPRESS_REGION }}
             storage-class: "EXPRESS_ONEZONE"
+            endpoint-url: ""
     permissions:
       id-token: write
       contents: read
@@ -95,6 +97,7 @@ jobs:
           CI_REGION=${{ matrix.test-run.region }} \
           CI_BUCKET=${{ matrix.test-run.bucket }} \
           CI_STORAGE_CLASS=${{ matrix.test-run.storage-class }} \
+          CI_CUSTOM_ENDPOINT_URL=${{ matrix.test-run.endpoint-url }} \
           pytest s3torchconnectorclient/python/tst/integration -n auto
 
       - name: Save Cargo cache

run_cibuildwheel_on_ec2.sh

@@ -10,6 +10,7 @@ BUCKET_NAME=$3
 PREFIX=$4
 EXPRESS_REGION_NAME=$5
 EXPRESS_BUCKET_NAME=$6
+S3_CUSTOM_ENDPOINT_URL=$7
 
 FILE_NAME="tmp_cred.json"
 # Set metadata token TTL to 6 hours
@@ -28,5 +29,6 @@ export S3_BUCKET=${BUCKET_NAME}
 export S3_PREFIX=${PREFIX}
 export S3_EXPRESS_REGION=${EXPRESS_REGION_NAME}
 export S3_EXPRESS_BUCKET=${EXPRESS_BUCKET_NAME}
+export S3_CUSTOM_ENDPOINT_URL=${S3_CUSTOM_ENDPOINT_URL}
 
 cibuildwheel --output-dir wheelhouse --platform linux s3torchconnectorclient

s3torchconnectorclient/pyproject.toml

@@ -47,10 +47,10 @@ test-extras = "test"
 test-command = [
     "pytest {package}/python/tst/unit",
     "pytest {package}/../s3torchconnector/tst/unit",
-    "CI_STORAGE_CLASS='' CI_REGION=${S3_REGION} CI_BUCKET=${S3_BUCKET} CI_PREFIX=${S3_PREFIX} pytest {package}/python/tst/integration",
-    "CI_STORAGE_CLASS='' CI_REGION=${S3_REGION} CI_BUCKET=${S3_BUCKET} CI_PREFIX=${S3_PREFIX} pytest {package}/../s3torchconnector/tst/e2e",
-    "CI_STORAGE_CLASS=EXPRESS_ONEZONE CI_REGION=${S3_EXPRESS_REGION} CI_BUCKET=${S3_EXPRESS_BUCKET} CI_PREFIX=${S3_PREFIX} pytest {package}/python/tst/integration",
-    "CI_STORAGE_CLASS=EXPRESS_ONEZONE CI_REGION=${S3_EXPRESS_REGION} CI_BUCKET=${S3_EXPRESS_BUCKET} CI_PREFIX=${S3_PREFIX} pytest {package}/../s3torchconnector/tst/e2e",
+    "CI_STORAGE_CLASS='' CI_REGION=${S3_REGION} CI_BUCKET=${S3_BUCKET} CI_PREFIX=${S3_PREFIX} CI_CUSTOM_ENDPOINT_URL=${S3_CUSTOM_ENDPOINT_URL} pytest {package}/python/tst/integration",
+    "CI_STORAGE_CLASS='' CI_REGION=${S3_REGION} CI_BUCKET=${S3_BUCKET} CI_PREFIX=${S3_PREFIX} CI_CUSTOM_ENDPOINT_URL=${S3_CUSTOM_ENDPOINT_URL} pytest {package}/../s3torchconnector/tst/e2e",
+    "CI_STORAGE_CLASS=EXPRESS_ONEZONE CI_REGION=${S3_EXPRESS_REGION} CI_BUCKET=${S3_EXPRESS_BUCKET} CI_PREFIX=${S3_PREFIX} CI_CUSTOM_ENDPOINT_URL='' pytest {package}/python/tst/integration",
+    "CI_STORAGE_CLASS=EXPRESS_ONEZONE CI_REGION=${S3_EXPRESS_REGION} CI_BUCKET=${S3_EXPRESS_BUCKET} CI_PREFIX=${S3_PREFIX} CI_CUSTOM_ENDPOINT_URL='' pytest {package}/../s3torchconnector/tst/e2e",
 ]
 environment-pass = [
     "AWS_ACCESS_KEY_ID",
@@ -60,7 +60,8 @@ environment-pass = [
     "S3_BUCKET",
     "S3_PREFIX",
     "S3_EXPRESS_BUCKET",
-    "S3_EXPRESS_REGION"
+    "S3_EXPRESS_REGION",
+    "S3_CUSTOM_ENDPOINT_URL"
 ]
 before-build = "cp README.md s3torchconnectorclient; cp LICENSE s3torchconnectorclient/; cp THIRD-PARTY-LICENSES s3torchconnectorclient/; cp NOTICE s3torchconnectorclient/"
 build = ["cp38*", "cp39*", "cp310*", "cp311*"]

s3torchconnectorclient/python/tst/integration/conftest.py

@@ -26,14 +26,21 @@ class BucketPrefixFixture(object):
     bucket: str
     prefix: str
     storage_class: str = None
+    endpoint_url: str = None
 
     def __init__(
-        self, region: str, bucket: str, prefix: str, storage_class: str = None
+        self,
+        region: str,
+        bucket: str,
+        prefix: str,
+        storage_class: str = None,
+        endpoint_url: str = None,
     ):
         self.bucket = bucket
         self.prefix = prefix
         self.region = region
         self.storage_class = storage_class
+        self.endpoint_url = endpoint_url
         self.contents = {}
         session = boto3.Session(region_name=region)
         self.s3 = session.client("s3")
@@ -61,12 +68,13 @@ def get_test_bucket_prefix(name: str) -> BucketPrefixFixture:
     prefix = getenv("CI_PREFIX")
     region = getenv("CI_REGION")
     storage_class = getenv("CI_STORAGE_CLASS", optional=True)
+    endpoint_url = getenv("CI_CUSTOM_ENDPOINT_URL", optional=True)
     assert prefix == "" or prefix.endswith("/")
 
     nonce = random.randrange(2**64)
     prefix = f"{prefix}{name}/{nonce}/"
 
-    return BucketPrefixFixture(region, bucket, prefix, storage_class)
+    return BucketPrefixFixture(region, bucket, prefix, storage_class, endpoint_url)
 
 
 @pytest.fixture

s3torchconnectorclient/python/tst/integration/test_mountpoint_s3_integration.py

@@ -40,7 +40,7 @@ def test_get_object_with_endpoint(sample_directory):
     client = MountpointS3Client(
         sample_directory.region,
         TEST_USER_AGENT_PREFIX,
-        endpoint="http://s3.amazonaws.com",
+        endpoint=sample_directory.endpoint_url,
     )
     stream = client.get_object(
         sample_directory.bucket, f"{sample_directory.prefix}hello_world.txt"