Merge pull request #742 from Altinity/feature/lazy_load_metadata

Make DataLake metadata more lazy

Author: Enmk

src/Disks/ObjectStorages/IObjectStorage.cpp

@@ -148,4 +148,18 @@ std::string RelativePathWithMetadata::CommandInTaskResponse::to_string() const
     return oss.str();
 }
 
+
+void RelativePathWithMetadata::loadMetadata(ObjectStoragePtr object_storage)
+{
+    if (!metadata)
+    {
+        const auto & path = isArchive() ? getPathToArchive() : getPath();
+
+        if (query_settings.ignore_non_existent_file)
+            metadata = object_storage->tryGetObjectMetadata(path);
+        else
+            metadata = object_storage->getObjectMetadata(path);
+    }
+}
+
 }

src/Disks/ObjectStorages/IObjectStorage.h

@@ -174,6 +174,8 @@ struct RelativePathWithMetadata
     std::optional<DataFileMetaInfoPtr> getFileMetaInfo() const { return file_meta_info; }
 
     const CommandInTaskResponse & getCommand() const { return command; }
+
+    void loadMetadata(ObjectStoragePtr object_storage);
 };
 
 struct ObjectKeyWithMetadata

src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.cpp

@@ -50,12 +50,16 @@ class KeysIterator : public IObjectIterator
                 return nullptr;
 
             auto key = data_files[current_index];
-            auto object_metadata = object_storage->getObjectMetadata(key);
 
             if (callback)
-                callback(FileProgress(0, object_metadata.size_bytes));
-
-            return std::make_shared<ObjectInfo>(key, std::move(object_metadata));
+            {
+                /// Too expencive to load size for metadata always
+                /// because it requires API call to external storage.
+                /// In many cases only keys are needed.
+                callback(FileProgress(0, 1));
+            }
+
+            return std::make_shared<ObjectInfo>(key, std::nullopt);
         }
     }
 

src/Storages/ObjectStorage/ReadBufferIterator.cpp

@@ -76,10 +76,7 @@ std::optional<ColumnsDescription> ReadBufferIterator::tryGetColumnsFromCache(
         const auto & object_info = (*it);
         auto get_last_mod_time = [&] -> std::optional<time_t>
         {
-            const auto & path = object_info->isArchive() ? object_info->getPathToArchive() : object_info->getPath();
-            if (!object_info->metadata)
-                object_info->metadata = object_storage->tryGetObjectMetadata(path);
-
+            object_info->loadMetadata(object_storage);
             return object_info->metadata
                 ? std::optional<time_t>(object_info->metadata->last_modified.epochTime())
                 : std::nullopt;
@@ -151,7 +148,6 @@ std::unique_ptr<ReadBuffer> ReadBufferIterator::recreateLastReadBuffer()
 {
     auto context = getContext();
 
-    const auto & path = current_object_info->isArchive() ? current_object_info->getPathToArchive() : current_object_info->getPath();
     auto impl = createReadBuffer(*current_object_info, object_storage, context, getLogger("ReadBufferIterator"));
 
     const auto compression_method = chooseCompressionMethod(current_object_info->getFileName(), configuration->getCompressionMethod());
@@ -250,6 +246,8 @@ ReadBufferIterator::Data ReadBufferIterator::next()
             prev_read_keys_size = read_keys.size();
         }
 
+        current_object_info->loadMetadata(object_storage);
+
         if (query_settings.skip_empty_files
             && current_object_info->metadata && current_object_info->metadata->size_bytes == 0)
             continue;

src/Storages/ObjectStorage/StorageObjectStorageSource.cpp

@@ -511,21 +511,7 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade
         if (object_info->getPath().empty())
             return {};
 
-        if (!object_info->metadata)
-        {
-            const auto & path = object_info->isArchive() ? object_info->getPathToArchive() : object_info->getPath();
-
-            if (query_settings.ignore_non_existent_file)
-            {
-                auto metadata = object_storage->tryGetObjectMetadata(path);
-                if (!metadata)
-                    return {};
-
-                object_info->metadata = metadata;
-            }
-            else
-                object_info->metadata = object_storage->getObjectMetadata(path);
-        }
+        object_info->loadMetadata(object_storage, query_settings.ignore_non_existent_file);
     }
     while (not_a_path || (query_settings.skip_empty_files && object_info->metadata->size_bytes == 0));