Move json serialization under condition in all insertRowToLogTable calls

Author: ianton-ru

src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp

@@ -185,9 +185,12 @@ void AzureObjectStorage::listObjects(const std::string & path, RelativePathsWith
         ProfileEvents::increment(ProfileEvents::AzureListObjects);
         if (client_ptr->IsClientForDisk())
             ProfileEvents::increment(ProfileEvents::DiskAzureListObjects);
-        ProfileEventTimeIncrement<Microseconds> watch(ProfileEvents::AzureListObjectsMicroseconds);
 
-        blob_list_response = client_ptr->ListBlobs(options);
+        {
+            ProfileEventTimeIncrement<Microseconds> watch(ProfileEvents::AzureListObjectsMicroseconds);
+            blob_list_response = client_ptr->ListBlobs(options);
+        }
+
         const auto & blobs_list = blob_list_response.Blobs;
 
         for (const auto & blob : blobs_list)

src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp

@@ -138,9 +138,13 @@ class S3IteratorAsync final : public IObjectStorageIteratorAsync
     {
         ProfileEvents::increment(ProfileEvents::S3ListObjects);
         ProfileEvents::increment(ProfileEvents::DiskS3ListObjects);
-        ProfileEventTimeIncrement<Microseconds> watch(ProfileEvents::S3ListObjectsMicroseconds);
 
-        auto outcome = client->ListObjectsV2(*request);
+        Aws::S3::Model::ListObjectsV2Outcome outcome;
+
+        {
+            ProfileEventTimeIncrement<Microseconds> watch(ProfileEvents::S3ListObjectsMicroseconds);
+            outcome = client->ListObjectsV2(*request);
+        }
 
         /// Outcome failure will be handled on the caller side.
         if (outcome.IsSuccess())
@@ -266,9 +270,12 @@ void S3ObjectStorage::listObjects(const std::string & path, RelativePathsWithMet
     {
         ProfileEvents::increment(ProfileEvents::S3ListObjects);
         ProfileEvents::increment(ProfileEvents::DiskS3ListObjects);
-        ProfileEventTimeIncrement<Microseconds> watch(ProfileEvents::S3ListObjectsMicroseconds);
 
-        outcome = client.get()->ListObjectsV2(request);
+        {
+            ProfileEventTimeIncrement<Microseconds> watch(ProfileEvents::S3ListObjectsMicroseconds);
+            outcome = client.get()->ListObjectsV2(request);
+        }
+
         throwIfError(outcome);
 
         auto result = outcome.GetResult();

src/Interpreters/IcebergMetadataLog.cpp

@@ -84,7 +84,7 @@ void IcebergMetadataLogElement::appendToBlock(MutableColumns & columns) const
 
 void insertRowToLogTable(
     const ContextPtr & local_context,
-    String row,
+    std::function<String()> get_row,
     IcebergMetadataLogLevel row_log_level,
     const String & table_path,
     const String & file_path,
@@ -104,40 +104,7 @@ void insertRowToLogTable(
             .content_type = row_log_level,
             .table_path = table_path,
             .file_path = file_path,
-            .metadata_content = row,
-            .row_in_file = row_in_file});
-}
-
-String dumpMetadataObjectToString(const Poco::JSON::Object::Ptr & metadata_object)
-{
-    std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM
-    Poco::JSON::Stringifier::stringify(metadata_object, oss);
-    return removeEscapedSlashes(oss.str());
-}
-
-void insertRowToLogTable(
-    const ContextPtr & local_context,
-    const Poco::JSON::Object::Ptr metadata_object,
-    IcebergMetadataLogLevel row_log_level,
-    const String & table_path,
-    const String & file_path,
-    std::optional<UInt64> row_in_file)
-{
-    IcebergMetadataLogLevel set_log_level = local_context->getSettingsRef()[Setting::iceberg_metadata_log_level].value;
-    if (set_log_level < row_log_level)
-        return;
-    timespec spec{};
-    if (clock_gettime(CLOCK_REALTIME, &spec))
-        throw ErrnoException(ErrorCodes::CANNOT_CLOCK_GETTIME, "Cannot clock_gettime");
-
-    Context::getGlobalContextInstance()->getIcebergMetadataLog()->add(
-        DB::IcebergMetadataLogElement{
-            .current_time = spec.tv_sec,
-            .query_id = local_context->getCurrentQueryId(),
-            .content_type = row_log_level,
-            .table_path = table_path,
-            .file_path = file_path,
-            .metadata_content = dumpMetadataObjectToString(metadata_object),
+            .metadata_content = get_row(),
             .row_in_file = row_in_file});
 }
 }

src/Interpreters/IcebergMetadataLog.h

@@ -26,15 +26,7 @@ struct IcebergMetadataLogElement
 
 void insertRowToLogTable(
     const ContextPtr & local_context,
-    String row,
-    IcebergMetadataLogLevel row_log_level,
-    const String & table_path,
-    const String & file_path,
-    std::optional<UInt64> row_in_file);
-
-void insertRowToLogTable(
-    const ContextPtr & local_context,
-    const Poco::JSON::Object::Ptr metadata_object,
+    std::function<String()> get_row,
     IcebergMetadataLogLevel row_log_level,
     const String & table_path,
     const String & file_path,

src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp

@@ -110,6 +110,16 @@ extern const SettingsBool allow_experimental_insert_into_iceberg;
 extern const SettingsBool allow_experimental_iceberg_compaction;
 }
 
+namespace
+{
+String dumpMetadataObjectToString(const Poco::JSON::Object::Ptr & metadata_object)
+{
+    std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM
+    Poco::JSON::Stringifier::stringify(metadata_object, oss);
+    return removeEscapedSlashes(oss.str());
+}
+}
+
 
 using namespace Iceberg;
 
@@ -233,9 +243,10 @@ bool IcebergMetadata::update(const ContextPtr & local_context)
 
     updateState(local_context, metadata_object);
 
+    auto dump_metadata = [&]()->String { return dumpMetadataObjectToString(metadata_object); };
     insertRowToLogTable(
         local_context,
-        metadata_object,
+        dump_metadata,
         DB::IcebergMetadataLogLevel::Metadata,
         configuration_ptr->getRawPath().path,
         metadata_file_path,
@@ -763,9 +774,10 @@ DataLakeMetadataPtr IcebergMetadata::create(
 
     auto format_version = object->getValue<int>(f_format_version);
 
+    auto dump_metadata = [&]()->String { return dumpMetadataObjectToString(object); };
     insertRowToLogTable(
         local_context,
-        object,
+        dump_metadata,
         DB::IcebergMetadataLogLevel::Metadata,
         configuration_ptr->getRawPath().path,
         metadata_file_path,

src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp

@@ -155,9 +155,10 @@ ManifestFileContent::ManifestFileContent(
     const String & path_to_manifest_file_)
     : path_to_manifest_file(path_to_manifest_file_)
 {
+    auto dump_metadata = [&]()->String { return manifest_file_deserializer.getMetadataContent(); };
     insertRowToLogTable(
         context,
-        manifest_file_deserializer.getMetadataContent(),
+        dump_metadata,
         DB::IcebergMetadataLogLevel::ManifestFileMetadata,
         common_path,
         path_to_manifest_file,
@@ -230,9 +231,10 @@ ManifestFileContent::ManifestFileContent(
 
     for (size_t i = 0; i < manifest_file_deserializer.rows(); ++i)
     {
+        auto dump_row_metadata = [&]()->String { return manifest_file_deserializer.getContent(i); };
         insertRowToLogTable(
             context,
-            manifest_file_deserializer.getContent(i),
+            dump_row_metadata,
             DB::IcebergMetadataLogLevel::ManifestFileEntry,
             common_path,
             path_to_manifest_file,

src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.cpp

@@ -148,9 +148,10 @@ ManifestFileCacheKeys getManifestList(
 
         ManifestFileCacheKeys manifest_file_cache_keys;
 
+        auto dump_metadata = [&]()->String { return manifest_list_deserializer.getMetadataContent(); };
         insertRowToLogTable(
             local_context,
-            manifest_list_deserializer.getMetadataContent(),
+            dump_metadata,
             DB::IcebergMetadataLogLevel::ManifestListMetadata,
             configuration_ptr->getRawPath().path,
             filename,
@@ -185,9 +186,10 @@ ManifestFileCacheKeys getManifestList(
             manifest_file_cache_keys.emplace_back(
                 manifest_file_name, added_sequence_number, added_snapshot_id.safeGet<Int64>(), content_type);
 
+            auto dump_row_metadata = [&]()->String { return manifest_list_deserializer.getContent(i); };
             insertRowToLogTable(
                 local_context,
-                manifest_list_deserializer.getContent(i),
+                dump_row_metadata,
                 DB::IcebergMetadataLogLevel::ManifestListEntry,
                 configuration_ptr->getRawPath().path,
                 filename,