Skip to content

Commit 2e73a41

Browse files
guitcastroguilhermecastroFokko
authored
Support s3.signer.endpoint for the REST catalog (apache#1029)
* s3_signer_endpoint * prune any trailing whitespaces Co-authored-by: Fokko Driesprong <[email protected]> * fallback to default value instead of "endpoint" property Co-authored-by: Fokko Driesprong <[email protected]> * fix test_s3v4_rest_signer_endpoint * Fix missing backtick Co-authored-by: Fokko Driesprong <[email protected]> * rename S3_SIGNER_ENDPOINT_DEFAULT_VALUE to S3_SIGNER_ENDPOINT_DEFAULT * fix s3.signer.endpoint docs * fk typo in signer * fix fmt --------- Co-authored-by: guilhermecastro <[email protected]> Co-authored-by: Fokko Driesprong <[email protected]>
1 parent 4f33f3a commit 2e73a41

File tree

4 files changed

+65
-5
lines changed

4 files changed

+65
-5
lines changed

‎mkdocs/docs/configuration.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,8 @@ For the FileIO there are several configuration options available:
8282
| s3.secret-access-key | password | Configure the static secret access key used to access the FileIO. |
8383
| s3.session-token | AQoDYXdzEJr... | Configure the static session token used to access the FileIO. |
8484
| s3.signer | bearer | Configure the signature version of the FileIO. |
85-
| s3.signer.uri | http://my.signer:8080/s3 | Configure the remote signing uri if it differs from the catalog uri. Remote signing is only implemented for `FsspecFileIO`. The final request is sent to `<s3.singer.uri>/v1/aws/s3/sign`. |
85+
| s3.signer.uri | http://my.signer:8080/s3 | Configure the remote signing uri if it differs from the catalog uri. Remote signing is only implemented for `FsspecFileIO`. The final request is sent to `<s3.signer.uri>/<s3.signer.endpoint>`. |
86+
| s3.signer.endpoint | v1/main/s3-sign | Configure the remote signing endpoint. Remote signing is only implemented for `FsspecFileIO`. The final request is sent to `<s3.signer.uri>/<s3.signer.endpoint>`. (default : v1/aws/s3/sign). |
8687
| s3.region | us-west-2 | Sets the region of the bucket |
8788
| s3.proxy-uri | http://my.proxy.com:8080 | Configure the proxy server to be used by the FileIO. |
8889
| s3.connect-timeout | 60.0 | Configure socket connection timeout, in seconds. |

‎pyiceberg/io/__init__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,8 @@
5858
S3_PROXY_URI = "s3.proxy-uri"
5959
S3_CONNECT_TIMEOUT = "s3.connect-timeout"
6060
S3_SIGNER_URI = "s3.signer.uri"
61+
S3_SIGNER_ENDPOINT = "s3.signer.endpoint"
62+
S3_SIGNER_ENDPOINT_DEFAULT = "v1/aws/s3/sign"
6163
HDFS_HOST = "hdfs.host"
6264
HDFS_PORT = "hdfs.port"
6365
HDFS_USER = "hdfs.user"

‎pyiceberg/io/fsspec.py

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@
6767
S3_REGION,
6868
S3_SECRET_ACCESS_KEY,
6969
S3_SESSION_TOKEN,
70+
S3_SIGNER_ENDPOINT,
71+
S3_SIGNER_ENDPOINT_DEFAULT,
7072
S3_SIGNER_URI,
7173
ADLFS_ClIENT_SECRET,
7274
FileIO,
@@ -86,6 +88,8 @@ def s3v4_rest_signer(properties: Properties, request: AWSRequest, **_: Any) -> A
8688
raise SignError("Signer set, but token is not available")
8789

8890
signer_url = properties.get(S3_SIGNER_URI, properties["uri"]).rstrip("/")
91+
signer_endpoint = properties.get(S3_SIGNER_ENDPOINT, S3_SIGNER_ENDPOINT_DEFAULT)
92+
8993
signer_headers = {"Authorization": f"Bearer {properties[TOKEN]}"}
9094
signer_body = {
9195
"method": request.method,
@@ -94,7 +98,7 @@ def s3v4_rest_signer(properties: Properties, request: AWSRequest, **_: Any) -> A
9498
"headers": {key: [val] for key, val in request.headers.items()},
9599
}
96100

97-
response = requests.post(f"{signer_url}/v1/aws/s3/sign", headers=signer_headers, json=signer_body)
101+
response = requests.post(f"{signer_url}/{signer_endpoint.strip()}", headers=signer_headers, json=signer_body)
98102
try:
99103
response.raise_for_status()
100104
response_json = response.json()
@@ -131,9 +135,9 @@ def _s3(properties: Properties) -> AbstractFileSystem:
131135

132136
if signer := properties.get("s3.signer"):
133137
logger.info("Loading signer %s", signer)
134-
if singer_func := SIGNERS.get(signer):
135-
singer_func_with_properties = partial(singer_func, properties)
136-
register_events["before-sign.s3"] = singer_func_with_properties
138+
if signer_func := SIGNERS.get(signer):
139+
signer_func_with_properties = partial(signer_func, properties)
140+
register_events["before-sign.s3"] = signer_func_with_properties
137141

138142
# Disable the AWS Signer
139143
config_kwargs["signature_version"] = UNSIGNED

‎tests/io/test_fsspec.py

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -727,6 +727,59 @@ def test_s3v4_rest_signer(requests_mock: Mocker) -> None:
727727
}
728728

729729

730+
def test_s3v4_rest_signer_endpoint(requests_mock: Mocker) -> None:
731+
new_uri = "https://other-bucket/metadata/snap-8048355899640248710-1-a5c8ea2d-aa1f-48e8-89f4-1fa69db8c742.avro"
732+
endpoint = "v1/main/s3-sign/foo.bar?e=e&b=b&k=k=k&s=s&w=w"
733+
requests_mock.post(
734+
f"{TEST_URI}/{endpoint}",
735+
json={
736+
"uri": new_uri,
737+
"headers": {
738+
"Authorization": [
739+
"AWS4-HMAC-SHA256 Credential=ASIAQPRZZYGHUT57DL3I/20221017/us-west-2/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=430582a17d61ab02c272896fa59195f277af4bdf2121c441685e589f044bbe02"
740+
],
741+
"Host": ["bucket.s3.us-west-2.amazonaws.com"],
742+
"User-Agent": ["Botocore/1.27.59 Python/3.10.7 Darwin/21.5.0"],
743+
"x-amz-content-sha256": ["UNSIGNED-PAYLOAD"],
744+
"X-Amz-Date": ["20221017T102940Z"],
745+
"X-Amz-Security-Token": [
746+
"YQoJb3JpZ2luX2VjEDoaCXVzLXdlc3QtMiJGMEQCID/fFxZP5oaEgQmcwP6XhZa0xSq9lmLSx8ffaWbySfUPAiAesa7sjd/WV4uwRTO0S03y/MWVtgpH+/NyZQ4bZgLVriqrAggTEAEaDDAzMzQwNzIyMjE1OSIMOeFOWhZIurMmAqjsKogCxMCqxX8ZjK0gacAkcDqBCyA7qTSLhdfKQIH/w7WpLBU1km+cRUWWCudan6gZsAq867DBaKEP7qI05DAWr9MChAkgUgyI8/G3Z23ET0gAedf3GsJbakB0F1kklx8jPmj4BPCht9RcTiXiJ5DxTS/cRCcalIQXmPFbaJSqpBusVG2EkWnm1v7VQrNPE2Os2b2P293vpbhwkyCEQiGRVva4Sw9D1sKvqSsK10QCRG+os6dFEOu1kARaXi6pStvR4OVmj7OYeAYjzaFchn7nz2CSae0M4IluiYQ01eQAywbfRo9DpKSmDM/DnPZWJnD/woLhaaaCrCxSSEaFsvGOHFhLd3Rknw1v0jADMILUtJoGOp4BpqKqyMz0CY3kpKL0jfR3ykTf/ge9wWVE0Alr7wRIkGCIURkhslGHqSyFRGoTqIXaxU+oPbwlw/0w/nYO7qQ6bTANOWye/wgw4h/NmJ6vU7wnZTXwREf1r6MF72++bE/fMk19LfVb8jN/qrUqAUXTc8gBAUxL5pgy8+oT/JnI2BkVrrLS4ilxEXP9Ahm+6GDUYXV4fBpqpZwdkzQ/5Gw="
747+
],
748+
},
749+
"extensions": {},
750+
},
751+
status_code=200,
752+
)
753+
754+
request = AWSRequest(
755+
method="HEAD",
756+
url="https://bucket/metadata/snap-8048355899640248710-1-a5c8ea2d-aa1f-48e8-89f4-1fa69db8c742.avro",
757+
headers={"User-Agent": "Botocore/1.27.59 Python/3.10.7 Darwin/21.5.0"},
758+
data=b"",
759+
params={},
760+
auth_path="/metadata/snap-8048355899640248710-1-a5c8ea2d-aa1f-48e8-89f4-1fa69db8c742.avro",
761+
)
762+
request.context = {
763+
"client_region": "us-west-2",
764+
"has_streaming_input": False,
765+
"auth_type": None,
766+
"signing": {"bucket": "bucket"},
767+
"retries": {"attempt": 1, "invocation-id": "75d143fb-0219-439b-872c-18213d1c8d54"},
768+
}
769+
770+
signed_request = s3v4_rest_signer({"token": "abc", "uri": TEST_URI, "s3.signer.endpoint": endpoint}, request)
771+
772+
assert signed_request.url == new_uri
773+
assert dict(signed_request.headers) == {
774+
"Authorization": "AWS4-HMAC-SHA256 Credential=ASIAQPRZZYGHUT57DL3I/20221017/us-west-2/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=430582a17d61ab02c272896fa59195f277af4bdf2121c441685e589f044bbe02",
775+
"Host": "bucket.s3.us-west-2.amazonaws.com",
776+
"User-Agent": "Botocore/1.27.59 Python/3.10.7 Darwin/21.5.0",
777+
"X-Amz-Date": "20221017T102940Z",
778+
"X-Amz-Security-Token": "YQoJb3JpZ2luX2VjEDoaCXVzLXdlc3QtMiJGMEQCID/fFxZP5oaEgQmcwP6XhZa0xSq9lmLSx8ffaWbySfUPAiAesa7sjd/WV4uwRTO0S03y/MWVtgpH+/NyZQ4bZgLVriqrAggTEAEaDDAzMzQwNzIyMjE1OSIMOeFOWhZIurMmAqjsKogCxMCqxX8ZjK0gacAkcDqBCyA7qTSLhdfKQIH/w7WpLBU1km+cRUWWCudan6gZsAq867DBaKEP7qI05DAWr9MChAkgUgyI8/G3Z23ET0gAedf3GsJbakB0F1kklx8jPmj4BPCht9RcTiXiJ5DxTS/cRCcalIQXmPFbaJSqpBusVG2EkWnm1v7VQrNPE2Os2b2P293vpbhwkyCEQiGRVva4Sw9D1sKvqSsK10QCRG+os6dFEOu1kARaXi6pStvR4OVmj7OYeAYjzaFchn7nz2CSae0M4IluiYQ01eQAywbfRo9DpKSmDM/DnPZWJnD/woLhaaaCrCxSSEaFsvGOHFhLd3Rknw1v0jADMILUtJoGOp4BpqKqyMz0CY3kpKL0jfR3ykTf/ge9wWVE0Alr7wRIkGCIURkhslGHqSyFRGoTqIXaxU+oPbwlw/0w/nYO7qQ6bTANOWye/wgw4h/NmJ6vU7wnZTXwREf1r6MF72++bE/fMk19LfVb8jN/qrUqAUXTc8gBAUxL5pgy8+oT/JnI2BkVrrLS4ilxEXP9Ahm+6GDUYXV4fBpqpZwdkzQ/5Gw=",
779+
"x-amz-content-sha256": "UNSIGNED-PAYLOAD",
780+
}
781+
782+
730783
def test_s3v4_rest_signer_forbidden(requests_mock: Mocker) -> None:
731784
requests_mock.post(
732785
f"{TEST_URI}/v1/aws/s3/sign",

0 commit comments

Comments
 (0)