parse s3 uri differently

Author: zvonand

src/Storages/ObjectStorage/Utils.cpp

@@ -3,6 +3,8 @@
 #include <Disks/ObjectStorages/ObjectStorageFactory.h>
 #include <Storages/ObjectStorage/StorageObjectStorage.h>
 #include <Poco/Util/MapConfiguration.h>
+#include <IO/S3/URI.h>
+
 
 namespace DB
 {
@@ -282,6 +284,76 @@ std::pair<DB::ObjectStoragePtr, std::string> resolveObjectStorageForPath(
     const std::string base_norm = normalizeSchema(base.scheme);
     const std::string target_norm = normalizeSchema(target.scheme);
 
+    // For S3 URIs, use S3::URI to properly URLs: https://s3.amazonaws.com/.....
+    #if USE_AWS_S3
+    if (target_norm == "s3" || target_norm == "https" || target_norm == "http")
+    {
+        try
+        {
+            S3::URI s3_uri(path);
+
+            if (base_norm == "s3" || base_norm == "https" || base_norm == "http")
+            {
+                S3::URI base_s3_uri(table_location);
+
+                if (s3_uri.bucket == base_s3_uri.bucket && s3_uri.endpoint == base_s3_uri.endpoint)
+                {
+                    if (base_storage)
+                        std::cerr << "\nSame S3 location, returning base, which is: " << base_storage->getName() << ", " << base_storage->getObjectsNamespace() << "\n";
+                    else
+                        std::cerr << "\nSame S3 location, returning base, which is: NULL\n";
+                    return {base_storage, s3_uri.key};
+                }
+                else
+                {
+                    std::cerr << "\nS3 storages are different:\n" << base_s3_uri.bucket << " @ " << base_s3_uri.endpoint
+                              << " vs " << s3_uri.bucket << " @ " << s3_uri.endpoint << "\n";
+                }
+            }
+
+            const std::string cache_key = "s3://" + s3_uri.bucket + "@" + (s3_uri.endpoint.empty() ? "amazonaws.com" : s3_uri.endpoint);
+
+            if (auto it = secondary_storages.find(cache_key); it != secondary_storages.end())
+            {
+                if (it->second)
+                    std::cerr << "\nfound in cache: " << it->second->getName() << ", " << it->second->getObjectsNamespace() << "\n";
+                else
+                    std::cerr << "\nfound in cache: NULL\n";
+                return {it->second, s3_uri.key};
+            }
+
+            /// TODO: maybe do not invent new configuration. Use old one and clean up later
+            Poco::AutoPtr<Poco::Util::MapConfiguration> cfg(new Poco::Util::MapConfiguration);
+
+            const std::string config_prefix = "object_storages." + cache_key;
+
+            cfg->setString(config_prefix + ".object_storage_type", "s3");
+
+            // Use the full endpoint or construct it from bucket
+            std::string endpoint = s3_uri.endpoint.empty()
+                ? ("https://" + s3_uri.bucket + ".s3.amazonaws.com")
+                : s3_uri.endpoint;
+            cfg->setString(config_prefix + ".endpoint", endpoint);
+
+            auto & factory = DB::ObjectStorageFactory::instance();
+
+            DB::ObjectStoragePtr storage = factory.create(cache_key, *cfg, config_prefix, context, /*skip_access_check*/ true);
+
+            secondary_storages.emplace(cache_key, storage);
+            if (storage)
+                std::cerr << "\ncreated new S3 storage: " << storage->getName() << ", " << storage->getObjectsNamespace() << ":\ndescr: " << storage->getDescription() << "\n";
+            else
+                std::cerr << "\ncreated new S3 storage: it is NULL\n";
+            return {storage, s3_uri.key};
+        }
+        catch (...)
+        {
+            // If S3::URI parsing fails, fall back to the old logic
+        }
+    }
+    #endif
+
+
     // Reuse base storage if scheme and authority (bucket) matches
     if (base_norm == target_norm && base.authority == target.authority)
     {