Skip to content

Commit 159b2f3

Browse files
authored
use anon argument when configuring s3fs.S3FileSystem (#2392)
<!-- Thanks for opening a pull request! --> <!-- In the case this PR will resolve an issue, please replace ${GITHUB_ISSUE_ID} below with the actual Github issue id. --> <!-- Closes #${GITHUB_ISSUE_ID} --> # Rationale for this change `s3fs.S3FileSystem` requires that we pass `anon` directly, it cannot be passed through `config_kwargs`. ## Are these changes tested? yes, unit tests updated. ## Are there any user-facing changes? no <!-- In the case of user-facing changes, please add the changelog label. -->
1 parent 7ad7056 commit 159b2f3

File tree

2 files changed

+8
-5
lines changed

2 files changed

+8
-5
lines changed

pyiceberg/io/fsspec.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -168,9 +168,11 @@ def _s3(properties: Properties) -> AbstractFileSystem:
168168
config_kwargs["read_timeout"] = float(request_timeout)
169169

170170
if s3_anonymous := properties.get(S3_ANONYMOUS):
171-
config_kwargs["anon"] = strtobool(s3_anonymous)
171+
anon = strtobool(s3_anonymous)
172+
else:
173+
anon = False
172174

173-
fs = S3FileSystem(client_kwargs=client_kwargs, config_kwargs=config_kwargs)
175+
fs = S3FileSystem(anon=anon, client_kwargs=client_kwargs, config_kwargs=config_kwargs)
174176

175177
for event_name, event_function in register_events.items():
176178
fs.s3.meta.events.unregister(event_name, unique_id=1925)

tests/io/test_fsspec.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,7 @@ def test_fsspec_s3_session_properties() -> None:
255255
s3_fileio.new_input(location=f"s3://warehouse/{filename}")
256256

257257
mock_s3fs.assert_called_with(
258+
anon=False,
258259
client_kwargs={
259260
"endpoint_url": "http://localhost:9000",
260261
"aws_access_key_id": "admin",
@@ -284,16 +285,15 @@ def test_fsspec_s3_session_properties_with_anonymous() -> None:
284285
s3_fileio.new_input(location=f"s3://warehouse/{filename}")
285286

286287
mock_s3fs.assert_called_with(
288+
anon=True,
287289
client_kwargs={
288290
"endpoint_url": "http://localhost:9000",
289291
"aws_access_key_id": "admin",
290292
"aws_secret_access_key": "password",
291293
"region_name": "us-east-1",
292294
"aws_session_token": "s3.session-token",
293295
},
294-
config_kwargs={
295-
"anon": True,
296-
},
296+
config_kwargs={},
297297
)
298298

299299

@@ -310,6 +310,7 @@ def test_fsspec_unified_session_properties() -> None:
310310
s3_fileio.new_input(location=f"s3://warehouse/{filename}")
311311

312312
mock_s3fs.assert_called_with(
313+
anon=False,
313314
client_kwargs={
314315
"endpoint_url": "http://localhost:9000",
315316
"aws_access_key_id": "client.access-key-id",

0 commit comments

Comments
 (0)