Skip to content

Commit 2624100

Browse files
authored
Support fsspec s3 addressing_style properties (#2517)
<!-- Thanks for opening a pull request! --> <!-- In the case this PR will resolve an issue, please replace ${GITHUB_ISSUE_ID} below with the actual Github issue id. --> <!-- Closes #${GITHUB_ISSUE_ID} --> # Rationale for this change Impl #2516 Then changed it like ```python catalog = load_catalog( "docs", **{ "type": "rest", "uri": "http://lakekeeper:8181/catalog", "s3.force-virtual-addressing": True , ... }, ) ``` ## Are these changes tested? Add test case for fsspec s3 addressing_style properties ## Are there any user-facing changes? The default behavior is not modified <!-- In the case of user-facing changes, please add the changelog label. -->
1 parent b63278b commit 2624100

File tree

2 files changed

+34
-0
lines changed

2 files changed

+34
-0
lines changed

pyiceberg/io/fsspec.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@
7070
S3_ANONYMOUS,
7171
S3_CONNECT_TIMEOUT,
7272
S3_ENDPOINT,
73+
S3_FORCE_VIRTUAL_ADDRESSING,
7374
S3_PROXY_URI,
7475
S3_REGION,
7576
S3_REQUEST_TIMEOUT,
@@ -168,6 +169,9 @@ def _s3(properties: Properties) -> AbstractFileSystem:
168169
if request_timeout := properties.get(S3_REQUEST_TIMEOUT):
169170
config_kwargs["read_timeout"] = float(request_timeout)
170171

172+
if _force_virtual_addressing := properties.get(S3_FORCE_VIRTUAL_ADDRESSING):
173+
config_kwargs["s3"] = {"addressing_style": "virtual"}
174+
171175
if s3_anonymous := properties.get(S3_ANONYMOUS):
172176
anon = strtobool(s3_anonymous)
173177
else:

tests/io/test_fsspec.py

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -306,6 +306,36 @@ def test_fsspec_s3_session_properties() -> None:
306306
)
307307

308308

309+
def test_fsspec_s3_session_properties_force_virtual_addressing() -> None:
310+
session_properties: Properties = {
311+
"s3.force-virtual-addressing": True,
312+
"s3.endpoint": "http://localhost:9000",
313+
"s3.access-key-id": "admin",
314+
"s3.secret-access-key": "password",
315+
"s3.region": "us-east-1",
316+
"s3.session-token": "s3.session-token",
317+
**UNIFIED_AWS_SESSION_PROPERTIES,
318+
}
319+
320+
with mock.patch("s3fs.S3FileSystem") as mock_s3fs:
321+
s3_fileio = FsspecFileIO(properties=session_properties)
322+
filename = str(uuid.uuid4())
323+
324+
s3_fileio.new_input(location=f"s3://warehouse/{filename}")
325+
326+
mock_s3fs.assert_called_with(
327+
anon=False,
328+
client_kwargs={
329+
"endpoint_url": "http://localhost:9000",
330+
"aws_access_key_id": "admin",
331+
"aws_secret_access_key": "password",
332+
"region_name": "us-east-1",
333+
"aws_session_token": "s3.session-token",
334+
},
335+
config_kwargs={"s3": {"addressing_style": "virtual"}},
336+
)
337+
338+
309339
def test_fsspec_s3_session_properties_with_anonymous() -> None:
310340
session_properties: Properties = {
311341
"s3.anonymous": "true",

0 commit comments

Comments
 (0)