rewind the part names logic

Author: arthurpassos

src/Storages/ObjectStorage/FilePathGenerator.h

@@ -10,15 +10,15 @@ namespace DB
     struct ObjectStorageFilePathGenerator
     {
         virtual ~ObjectStorageFilePathGenerator() = default;
-        virtual std::string getWritingPath(const std::string & partition_id, std::optional<std::string> filename_override = {}) const = 0;
+        virtual std::string getWritingPath(const std::string & partition_id) const = 0;
         virtual std::string getReadingPath() const = 0;
     };
 
     struct ObjectStorageWildcardFilePathGenerator : ObjectStorageFilePathGenerator
     {
         explicit ObjectStorageWildcardFilePathGenerator(const std::string & raw_path_) : raw_path(raw_path_) {}
 
-        std::string getWritingPath(const std::string & partition_id, std::optional<std::string> /**/ = {}) const override
+        std::string getWritingPath(const std::string & partition_id) const override
         {
             return PartitionedSink::replaceWildcards(raw_path, partition_id);
         }
@@ -41,9 +41,9 @@ namespace DB
             const std::shared_ptr<ObjectStorageFilenameGenerator> & filename_generator_)
         : raw_path(raw_path_), file_format(Poco::toLower(file_format_)), filename_generator(filename_generator_){}
 
-        std::string getWritingPath(const std::string & partition_id, std::optional<std::string> filename_override) const override
+        std::string getWritingPath(const std::string & partition_id) const override
         {
-            return raw_path + "/" + partition_id + "/"  + (filename_override ? *filename_override : filename_generator->generate()) + "." + file_format;
+            return raw_path + "/" + partition_id + "/"  + filename_generator->generate() + "." + file_format;
         }
 
         std::string getReadingPath() const override

src/Storages/ObjectStorage/StorageObjectStorage.cpp

@@ -554,9 +554,6 @@ void StorageObjectStorage::importMergeTreePartition(
     if (data_parts.empty())
         return;
 
-    RelativePathsWithMetadata relative_paths_with_metadata;
-    object_storage->listObjects(configuration->getRawPath().path, relative_paths_with_metadata, 1000);
-
     std::vector<QueryPlanPtr> part_plans;
     part_plans.reserve(data_parts.size());
 
@@ -580,44 +577,13 @@ void StorageObjectStorage::importMergeTreePartition(
     std::vector<StoredObject> files_to_be_deleted;
     for (const auto & data_part : data_parts)
     {
-        bool upload_part = true;
-        for (const auto & object_with_metadata : relative_paths_with_metadata)
-        {
-            const auto remote_object_filename = object_with_metadata->getFileNameWithoutExtension();
-            if (remote_object_filename == data_part->name)
-            {
-                upload_part = false;
-                break;
-            }
-
-            const auto remote_fake_part = MergeTreePartInfo::tryParsePartName(remote_object_filename, merge_tree_data.format_version);
-
-            if (!remote_fake_part)
-            {
-                continue;
-            }
-
-            /// If the part does not intersect, proceed to the next file
-            if (data_part->info.isDisjoint(remote_fake_part.value()))
-            {
-                continue;
-            }
-
-            files_to_be_deleted.emplace_back(object_with_metadata->relative_path);
-        }
-
-        if (!upload_part)
-        {
-            continue;
-        }
-
         const auto partition_columns = configuration->partition_strategy->getPartitionColumns();
 
         auto block_with_partition_values = data_part->partition.getBlockWithPartitionValues(partition_columns);
 
         const auto column_with_partition_key = configuration->partition_strategy->computePartitionKey(block_with_partition_values);
 
-        const auto file_path = configuration->file_path_generator->getWritingPath(column_with_partition_key->getDataAt(0).toString(), data_part->name);
+        const auto file_path = configuration->file_path_generator->getWritingPath(column_with_partition_key->getDataAt(0).toString());
 
         export_list_entries.emplace_back(local_context->getGlobalContext()->getExportsList().insert(
             merge_tree_data.getStorageID(),
@@ -683,9 +649,6 @@ void StorageObjectStorage::importMergeTreePartition(
     {
         root_pipeline.setNumThreads(local_context->getSettingsRef()[Setting::max_threads]);
 
-        /// shouldn't this be part of the sink and or pipeline?
-        object_storage->removeObjectsIfExist(files_to_be_deleted);
-
         CompletedPipelineExecutor exec(root_pipeline);
         exec.execute();
     }

src/Storages/ObjectStorage/StorageObjectStorageSink.cpp

@@ -133,16 +133,14 @@ PartitionedStorageObjectStorageSink::PartitionedStorageObjectStorageSink(
     const std::shared_ptr<ObjectStorageFilePathGenerator> & file_path_generator_,
     std::optional<FormatSettings> format_settings_,
     const Block & sample_block_,
-    ContextPtr context_,
-    std::optional<std::string> filename_override_)
+    ContextPtr context_)
     : object_storage(object_storage_)
     , configuration(configuration_)
     , file_path_generator(file_path_generator_)
     , query_settings(configuration_->getQuerySettings(context_))
     , format_settings(format_settings_)
     , sample_block(sample_block_)
     , context(context_)
-    , filename_override(filename_override_)
 {
 }
 
@@ -154,7 +152,7 @@ StorageObjectStorageSink::~StorageObjectStorageSink()
 
 SinkPtr PartitionedStorageObjectStorageSink::createSinkForPartition(const String & partition_id)
 {
-    auto file_path = file_path_generator->getWritingPath(partition_id, filename_override);
+    auto file_path = file_path_generator->getWritingPath(partition_id);
 
     validateNamespace(configuration->getNamespace(), configuration);
     validateKey(file_path);

src/Storages/ObjectStorage/StorageObjectStorageSink.h

@@ -50,8 +50,7 @@ class PartitionedStorageObjectStorageSink : public PartitionedSink::SinkCreator
         const std::shared_ptr<ObjectStorageFilePathGenerator> & file_path_generator_,
         std::optional<FormatSettings> format_settings_,
         const Block & sample_block_,
-        ContextPtr context_,
-        std::optional<std::string> filename_override_ = std::nullopt);
+        ContextPtr context_);
 
     SinkPtr createSinkForPartition(const String & partition_id) override;
 
@@ -64,7 +63,6 @@ class PartitionedStorageObjectStorageSink : public PartitionedSink::SinkCreator
     const std::optional<FormatSettings> format_settings;
     const Block sample_block;
     const ContextPtr context;
-    std::optional<std::string> filename_override;
 };
 
 }