Move S3 ambient credentials out of AccessConfig (#578)

Author: potter-potter

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.2.5
+
+**Fix**: move S3 ambient_credentials out of AccessConfig
+
 ## 1.2.4
 
 **Fix**: properly handle Together API 5xx errors as ProviderError instead of UserError

test/integration/connectors/test_s3.py

@@ -295,13 +295,13 @@ class TestS3AmbientCredentials:
 
     def test_ambient_credentials_field_default(self):
         """Test that ambient_credentials defaults to False"""
-        access_config = S3AccessConfig()
-        assert access_config.ambient_credentials is False
+        connection_config = S3ConnectionConfig()
+        assert connection_config.ambient_credentials is False
 
     def test_ambient_credentials_field_explicit(self):
         """Test setting ambient_credentials explicitly"""
-        access_config = S3AccessConfig(ambient_credentials=True)
-        assert access_config.ambient_credentials is True
+        connection_config = S3ConnectionConfig(ambient_credentials=True)
+        assert connection_config.ambient_credentials is True
 
     def test_default_blocks_automatic_credentials(self):
         """Test that default behavior blocks automatic credential pickup"""
@@ -341,8 +341,7 @@ def test_ambient_credentials_requires_env_var(self, monkeypatch):
         # Clear the environment variable
         monkeypatch.delenv("ALLOW_AMBIENT_CREDENTIALS_S3", raising=False)
 
-        access_config = S3AccessConfig(ambient_credentials=True)
-        connection_config = S3ConnectionConfig(access_config=access_config, anonymous=False)
+        connection_config = S3ConnectionConfig(ambient_credentials=True, anonymous=False)
 
         # Should raise error when env var is not set
         with pytest.raises(
@@ -356,8 +355,7 @@ def test_ambient_credentials_enables_ambient_mode(self, monkeypatch):
         # Set the environment variable
         monkeypatch.setenv("ALLOW_AMBIENT_CREDENTIALS_S3", "true")
 
-        access_config = S3AccessConfig(ambient_credentials=True)
-        connection_config = S3ConnectionConfig(access_config=access_config, anonymous=False)
+        connection_config = S3ConnectionConfig(ambient_credentials=True, anonymous=False)
 
         config = connection_config.get_access_config()
 
@@ -373,9 +371,11 @@ def test_ambient_credentials_field_excluded_from_config(self):
         access_config = S3AccessConfig(
             key="test-key",
             secret="test-secret",
-            ambient_credentials=True,  # Should be excluded
         )
-        connection_config = S3ConnectionConfig(access_config=access_config)
+        connection_config = S3ConnectionConfig(
+            access_config=access_config,
+            ambient_credentials=True,  # Should be excluded from final config
+        )
 
         config = connection_config.get_access_config()
 
@@ -412,8 +412,7 @@ def test_endpoint_url_preserved_with_all_auth_modes(self, monkeypatch):
 
         # Test with ambient credentials
         monkeypatch.setenv("ALLOW_AMBIENT_CREDENTIALS_S3", "true")
-        access_config = S3AccessConfig(ambient_credentials=True)
-        connection_config = S3ConnectionConfig(access_config=access_config, endpoint_url=endpoint)
+        connection_config = S3ConnectionConfig(ambient_credentials=True, endpoint_url=endpoint)
         config = connection_config.get_access_config()
         assert config["endpoint_url"] == endpoint
 
@@ -424,8 +423,7 @@ def test_endpoint_url_preserved_with_all_auth_modes(self, monkeypatch):
 
     def test_authentication_error_raised(self):
         """Test that authentication error is raised when automatic credentials would be used"""
-        access_config = S3AccessConfig(ambient_credentials=False)
-        connection_config = S3ConnectionConfig(access_config=access_config, anonymous=False)
+        connection_config = S3ConnectionConfig(ambient_credentials=False, anonymous=False)
 
         # This should raise UserAuthError with helpful message
         with pytest.raises(UserAuthError) as exc_info:
@@ -440,8 +438,7 @@ def test_ambient_credentials_env_var_variations(self, monkeypatch):
         """Test that only 'true' (case-insensitive) values for ALLOW_AMBIENT_CREDENTIALS_S3 work"""
         valid_values = ["true", "TRUE", "True", "tRuE"]
 
-        access_config = S3AccessConfig(ambient_credentials=True)
-        connection_config = S3ConnectionConfig(access_config=access_config, anonymous=False)
+        connection_config = S3ConnectionConfig(ambient_credentials=True, anonymous=False)
 
         for value in valid_values:
             monkeypatch.setenv("ALLOW_AMBIENT_CREDENTIALS_S3", value)
@@ -460,8 +457,7 @@ def test_ambient_credentials_info_logged(self, caplog, monkeypatch):
         # Ensure we capture INFO level logs
         caplog.set_level(logging.INFO)
 
-        access_config = S3AccessConfig(ambient_credentials=True)
-        connection_config = S3ConnectionConfig(access_config=access_config, anonymous=False)
+        connection_config = S3ConnectionConfig(ambient_credentials=True, anonymous=False)
 
         # This should trigger the ambient credentials info log
         config = connection_config.get_access_config()
@@ -491,7 +487,7 @@ async def test_s3_destination_with_ambient_credentials(self, upload_file: Path,
 
         # Use ambient credentials (no explicit key/secret provided)
         connection_config = S3ConnectionConfig(
-            access_config=S3AccessConfig(ambient_credentials=True),
+            ambient_credentials=True,
         )
         upload_config = S3UploaderConfig(remote_url=destination_path)
         uploader = S3Uploader(connection_config=connection_config, upload_config=upload_config)

test/integration/embedders/test_togetherai.py

@@ -27,6 +27,10 @@ def get_api_key() -> str:
     return api_key
 
 
+# TODO: re-enable these tests when TogetherAI API is fixed
+
+
+@pytest.mark.skip(reason="TogetherAI API disabled")
 @requires_env(API_KEY)
 def test_togetherai_embedder(embedder_file: Path):
     api_key = get_api_key()
@@ -40,6 +44,7 @@ def test_togetherai_embedder(embedder_file: Path):
     validate_embedding_output(original_elements=original_elements, output_elements=results)
 
 
+@pytest.mark.skip(reason="TogetherAI API disabled")
 @requires_env(API_KEY)
 def test_raw_togetherai_embedder(embedder_file: Path):
     api_key = get_api_key()
@@ -53,13 +58,15 @@ def test_raw_togetherai_embedder(embedder_file: Path):
     )
 
 
+@pytest.mark.skip(reason="TogetherAI API disabled")
 def test_raw_togetherai_embedder_invalid_credentials():
     embedder = TogetherAIEmbeddingEncoder(config=TogetherAIEmbeddingConfig(api_key="fake_api_key"))
 
     with pytest.raises(UserAuthError):
         embedder.get_exemplary_embedding()
 
 
+@pytest.mark.skip(reason="TogetherAI API disabled")
 @requires_env(API_KEY)
 @pytest.mark.asyncio
 async def test_raw_async_togetherai_embedder(embedder_file: Path):

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "1.2.4"  # pragma: no cover
+__version__ = "1.2.5"  # pragma: no cover

unstructured_ingest/processes/connectors/fsspec/s3.py

@@ -57,13 +57,6 @@ class S3AccessConfig(FsspecAccessConfig):
     token: Optional[str] = Field(
         default=None, description="If not anonymous, use this security token, if specified."
     )
-    ambient_credentials: bool = Field(
-        default=False,
-        description="Explicitly allow using ambient AWS credentials from .aws folder, "
-        "environment variables, or IAM roles. Requires ALLOW_AMBIENT_CREDENTIALS_S3 environment "
-        "variable to also be set to 'true' (case insensitive) for security. When False (default), "
-        "only explicit credentials or anonymous access are allowed.",
-    )
 
 
 class S3ConnectionConfig(FsspecConnectionConfig):
@@ -77,6 +70,13 @@ class S3ConnectionConfig(FsspecConnectionConfig):
     anonymous: bool = Field(
         default=False, description="Connect to s3 without local AWS credentials."
     )
+    ambient_credentials: bool = Field(
+        default=False,
+        description="Explicitly allow using ambient AWS credentials from .aws folder, "
+        "environment variables, or IAM roles. Requires ALLOW_AMBIENT_CREDENTIALS_S3 environment "
+        "variable to also be set to 'true' (case insensitive) for security. When False (default), "
+        "only explicit credentials or anonymous access are allowed.",
+    )
     connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
     def get_access_config(self) -> dict[str, Any]:
@@ -91,13 +91,9 @@ def get_access_config(self) -> dict[str, Any]:
             access_configs = {"anon": False}
             # Avoid injecting None by filtering out k,v pairs where the value is None
             access_configs.update(
-                {
-                    k: v
-                    for k, v in access_config.model_dump().items()
-                    if v is not None and k != "ambient_credentials"
-                }
+                {k: v for k, v in access_config.model_dump().items() if v is not None}
             )
-        elif access_config.ambient_credentials:
+        elif self.ambient_credentials:
             if os.getenv("ALLOW_AMBIENT_CREDENTIALS_S3", "").lower() == "true":
                 logger.info(
                     "Using ambient AWS credentials (environment variables, .aws folder, IAM roles)"