Merge branch 'customizations/24.3.11' into 24.3.5_aarch64-integration-tests

Author: MyroTk

src/IO/S3/URI.cpp

@@ -33,12 +33,17 @@ namespace S3
 URI::URI(const std::string & uri_)
 {
     /// Case when bucket name represented in domain name of S3 URL.
-    /// E.g. (https://bucket-name.s3.Region.amazonaws.com/key)
+    /// E.g. (https://bucket-name.s3.region.amazonaws.com/key)
     /// https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html#virtual-hosted-style-access
     static const RE2 virtual_hosted_style_pattern(R"((.+)\.(s3express[\-a-z0-9]+|s3|cos|obs|oss|eos)([.\-][a-z0-9\-.:]+))");
 
+    /// Case when AWS Private Link Interface is being used
+    /// E.g. (bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w.s3.us-east-1.vpce.amazonaws.com/bucket-name/key)
+    /// https://docs.aws.amazon.com/AmazonS3/latest/userguide/privatelink-interface-endpoints.html
+    static const RE2 aws_private_link_style_pattern(R"(bucket\.vpce\-([a-z0-9\-.]+)\.vpce.amazonaws.com(:\d{1,5})?)");
+
     /// Case when bucket name and key represented in path of S3 URL.
-    /// E.g. (https://s3.Region.amazonaws.com/bucket-name/key)
+    /// E.g. (https://s3.region.amazonaws.com/bucket-name/key)
     /// https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html#path-style-access
     static const RE2 path_style_pattern("^/([^/]*)/(.*)");
 
@@ -103,7 +108,10 @@ URI::URI(const std::string & uri_)
     String name;
     String endpoint_authority_from_uri;
 
-    if (re2::RE2::FullMatch(uri.getAuthority(), virtual_hosted_style_pattern, &bucket, &name, &endpoint_authority_from_uri))
+    bool is_using_aws_private_link_interface = re2::RE2::FullMatch(uri.getAuthority(), aws_private_link_style_pattern);
+
+    if (!is_using_aws_private_link_interface
+        && re2::RE2::FullMatch(uri.getAuthority(), virtual_hosted_style_pattern, &bucket, &name, &endpoint_authority_from_uri))
     {
         is_virtual_hosted_style = true;
         endpoint = uri.getScheme() + "://" + name + endpoint_authority_from_uri;

src/IO/S3/URI.h

@@ -17,6 +17,7 @@ namespace DB::S3
  * The following patterns are allowed:
  * s3://bucket/key
  * http(s)://endpoint/bucket/key
+ * http(s)://bucket.<vpce_endpoint_id>.s3.<region>.vpce.amazonaws.com<:port_number>/bucket_name/key
  */
 struct URI
 {

src/IO/tests/gtest_s3_uri.cpp

@@ -74,6 +74,40 @@ const TestCase TestCases[] = {
      "data",
      "",
      true},
+    {S3::URI("https://bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w.s3.us-east-1.vpce.amazonaws.com/root/nested/file.txt"),
+     "https://bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w.s3.us-east-1.vpce.amazonaws.com",
+     "root",
+     "nested/file.txt",
+     "",
+     false},
+    // Test with a file with no extension
+    {S3::URI("https://bucket.vpce-03b2c987f1bd55c5f-j3b4vg7w.s3.ap-southeast-2.vpce.amazonaws.com/some_bucket/document"),
+     "https://bucket.vpce-03b2c987f1bd55c5f-j3b4vg7w.s3.ap-southeast-2.vpce.amazonaws.com",
+     "some_bucket",
+     "document",
+     "",
+     false},
+    // Test with a deeply nested file path
+    {S3::URI("https://bucket.vpce-0242cd56f1bd55c5f-l5b7vg8x.s3.sa-east-1.vpce.amazonaws.com/some_bucket/b/c/d/e/f/g/h/i/j/data.json"),
+     "https://bucket.vpce-0242cd56f1bd55c5f-l5b7vg8x.s3.sa-east-1.vpce.amazonaws.com",
+     "some_bucket",
+     "b/c/d/e/f/g/h/i/j/data.json",
+     "",
+     false},
+    // Zonal
+    {S3::URI("https://bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w-us-east-1a.s3.us-east-1.vpce.amazonaws.com/root/nested/file.txt"),
+     "https://bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w-us-east-1a.s3.us-east-1.vpce.amazonaws.com",
+     "root",
+     "nested/file.txt",
+     "",
+     false},
+    // Non standard port
+    {S3::URI("https://bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w-us-east-1a.s3.us-east-1.vpce.amazonaws.com:65535/root/nested/file.txt"),
+     "https://bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w-us-east-1a.s3.us-east-1.vpce.amazonaws.com:65535",
+     "root",
+     "nested/file.txt",
+     "",
+     false},
 };
 
 class S3UriTest : public testing::TestWithParam<std::string>