Merge pull request ClickHouse#77406 from jkartseva/plain-rw-imitate-hardlink

Support ALTER TABLE ATTACH|DETACH|MOVE|REPLACE PARTITION for plain_rewritable disk

Author: jkartseva

src/Disks/IDisk.cpp

@@ -1,17 +1,18 @@
 #include "IDisk.h"
+#include <Core/Field.h>
+#include <Core/ServerUUID.h>
+#include <Disks/FakeDiskTransaction.h>
 #include <IO/ReadBufferFromFileBase.h>
 #include <IO/WriteBufferFromFileBase.h>
 #include <IO/copyData.h>
+#include <Interpreters/Context.h>
+#include <Storages/PartitionCommands.h>
 #include <Poco/Logger.h>
 #include <Poco/Util/AbstractConfiguration.h>
-#include <Interpreters/Context.h>
 #include <Common/ThreadPool.h>
-#include <Common/threadPoolCallbackRunner.h>
 #include <Common/logger_useful.h>
 #include <Common/setThreadName.h>
-#include <Core/Field.h>
-#include <Core/ServerUUID.h>
-#include <Disks/FakeDiskTransaction.h>
+#include <Common/threadPoolCallbackRunner.h>
 
 namespace CurrentMetrics
 {
@@ -191,6 +192,11 @@ void IDisk::truncateFile(const String &, size_t)
     throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Truncate operation is not implemented for disk of type {}", getDataSourceDescription().type);
 }
 
+bool IDisk::supportsPartitionCommand(const PartitionCommand & /*command*/) const
+{
+    return true;
+}
+
 SyncGuardPtr IDisk::getDirectorySyncGuard(const String & /* path */) const
 {
     return nullptr;

src/Disks/IDisk.h

@@ -72,6 +72,8 @@ using DiskObjectStoragePtr = std::shared_ptr<DiskObjectStorage>;
 
 using ObjectAttributes = std::map<std::string, std::string>;
 
+struct PartitionCommand;
+
 /**
  * Provide interface for reservation.
  */
@@ -444,6 +446,8 @@ class IDisk : public Space
 
     virtual bool supportsHardLinks() const { return true; }
 
+    virtual bool supportsPartitionCommand(const PartitionCommand & command) const;
+
     /// Check if disk is broken. Broken disks will have 0 space and cannot be used.
     virtual bool isBroken() const { return false; }
 

src/Disks/ObjectStorages/DiskObjectStorage.cpp

@@ -643,6 +643,11 @@ bool DiskObjectStorage::supportsHardLinks() const
     return !isWriteOnce() && !object_storage->isPlain();
 }
 
+bool DiskObjectStorage::supportsPartitionCommand(const PartitionCommand & command) const
+{
+    return !isWriteOnce() && metadata_storage->supportsPartitionCommand(command);
+}
+
 DiskObjectStoragePtr DiskObjectStorage::createDiskObjectStorage()
 {
     const auto config_prefix = "storage_configuration.disks." + name;

src/Disks/ObjectStorages/DiskObjectStorage.h

@@ -215,6 +215,8 @@ friend class DiskObjectStorageRemoteMetadataRestoreHelper;
 
     bool supportsHardLinks() const override;
 
+    bool supportsPartitionCommand(const PartitionCommand & command) const override;
+
     /// Get structure of object storage this disk works with. Examples:
     /// DiskObjectStorage(S3ObjectStorage)
     /// DiskObjectStorage(CachedObjectStorage(S3ObjectStorage))

src/Disks/ObjectStorages/IMetadataStorage.h

@@ -24,6 +24,7 @@ namespace ErrorCodes
 }
 
 class IMetadataStorage;
+struct PartitionCommand;
 
 /// Return the result of operation to the caller.
 /// It is used in `IDiskObjectStorageOperation::finalize` after metadata transaction executed to make decision on blob removal.
@@ -225,6 +226,8 @@ class IMetadataStorage : private boost::noncopyable
         throwNotImplemented();
     }
 
+    virtual bool supportsPartitionCommand(const PartitionCommand & /* command */) const = 0;
+
     virtual std::vector<std::string> listDirectory(const std::string & path) const = 0;
 
     virtual DirectoryIteratorPtr iterateDirectory(const std::string & path) const = 0;

src/Disks/ObjectStorages/InMemoryDirectoryPathMap.h

@@ -79,6 +79,20 @@ class InMemoryDirectoryPathMap
         return res;
     }
 
+    bool existsFile(const std::string & local_path) const
+    {
+        auto path = std::filesystem::path(local_path);
+        auto dir = path.parent_path();
+        auto filename = path.filename();
+
+        std::lock_guard lock(mutex);
+        auto it = map.find(dir);
+        if (it == map.end())
+            return false;
+
+        return it->second.files.contains(filename);
+    }
+
     bool addFile(const std::string & local_path)
     {
         auto path = std::filesystem::path(local_path);

src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp

@@ -1,7 +1,9 @@
-#include <Disks/ObjectStorages/MetadataStorageFromDisk.h>
 #include <Disks/ObjectStorages/IMetadataStorage.h>
-#include <IO/WriteHelpers.h>
+#include <Disks/ObjectStorages/MetadataStorageFromDisk.h>
 #include <IO/ReadHelpers.h>
+#include <IO/WriteHelpers.h>
+#include <Storages/PartitionCommands.h>
+
 #include <ranges>
 #include <shared_mutex>
 
@@ -44,6 +46,11 @@ time_t MetadataStorageFromDisk::getLastChanged(const std::string & path) const
     return disk->getLastChanged(path);
 }
 
+bool MetadataStorageFromDisk::supportsPartitionCommand(const PartitionCommand & /*command*/) const
+{
+    return true;
+}
+
 uint64_t MetadataStorageFromDisk::getFileSize(const String & path) const
 {
     auto metadata = readMetadata(path);

src/Disks/ObjectStorages/MetadataStorageFromDisk.h

@@ -54,6 +54,8 @@ class MetadataStorageFromDisk final : public IMetadataStorage
 
     bool supportsStat() const override { return disk->supportsStat(); }
 
+    bool supportsPartitionCommand(const PartitionCommand & command) const override;
+
     struct stat stat(const String & path) const override { return disk->stat(path); }
 
     std::vector<std::string> listDirectory(const std::string & path) const override;

src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp

@@ -4,6 +4,7 @@
 #include <Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h>
 #include <Disks/ObjectStorages/StaticDirectoryIterator.h>
 #include <Disks/ObjectStorages/StoredObject.h>
+#include <Storages/PartitionCommands.h>
 #include <Common/ObjectStorageKey.h>
 #include <Common/SipHash.h>
 #include <Common/logger_useful.h>
@@ -109,6 +110,11 @@ std::optional<Poco::Timestamp> MetadataStorageFromPlainObjectStorage::getLastMod
     return std::nullopt;
 }
 
+bool MetadataStorageFromPlainObjectStorage::supportsPartitionCommand(const PartitionCommand & /*command*/) const
+{
+    return false;
+}
+
 std::vector<std::string> MetadataStorageFromPlainObjectStorage::listDirectory(const std::string & path) const
 {
     auto key_prefix = object_storage->generateObjectKeyForPath(path, std::nullopt /* key_prefix */).serialize();
@@ -215,6 +221,26 @@ void MetadataStorageFromPlainObjectStorageTransaction::removeDirectory(const std
     }
 }
 
+void MetadataStorageFromPlainObjectStorageTransaction::createHardLink(const std::string & path_from, const std::string & path_to)
+{
+    if (metadata_storage.object_storage->isWriteOnce())
+        throwNotImplemented();
+
+    addOperation(std::make_unique<MetadataStorageFromPlainObjectStorageCopyFileOperation>(
+        path_from, path_to, *metadata_storage.getPathMap(), object_storage));
+}
+
+void MetadataStorageFromPlainObjectStorageTransaction::moveFile(const std::string & path_from, const std::string & path_to)
+{
+    if (metadata_storage.object_storage->isWriteOnce())
+        throwNotImplemented();
+
+    if (metadata_storage.existsDirectory(path_from))
+        moveDirectory(path_from, path_to);
+    else
+        throwNotImplemented();
+}
+
 void MetadataStorageFromPlainObjectStorageTransaction::createEmptyMetadataFile(const std::string & path)
 {
     if (metadata_storage.object_storage->isWriteOnce())

src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h

@@ -87,6 +87,7 @@ class MetadataStorageFromPlainObjectStorage : public IMetadataStorage
 
     bool supportsChmod() const override { return false; }
     bool supportsStat() const override { return false; }
+    bool supportsPartitionCommand(const PartitionCommand & command) const override;
 
 protected:
     /// Get the object storage prefix for storing metadata files.
@@ -117,14 +118,20 @@ class MetadataStorageFromPlainObjectStorageTransaction : public IMetadataTransac
 
     void addBlobToMetadata(const std::string & /* path */, ObjectStorageKey /* object_key */, uint64_t /* size_in_bytes */) override
     {
-        // Noop
+        /// Noop
     }
 
     void setLastModified(const String &, const Poco::Timestamp &) override
     {
         /// Noop
     }
 
+    /// Required for MergeTree backups.
+    void setReadOnly(const std::string & /*path*/) override
+    {
+        /// Noop
+    }
+
     void createEmptyMetadataFile(const std::string & /* path */) override;
 
     void createMetadataFile(const std::string & /* path */, ObjectStorageKey /* object_key */, uint64_t /* size_in_bytes */) override;
@@ -138,6 +145,11 @@ class MetadataStorageFromPlainObjectStorageTransaction : public IMetadataTransac
     void unlinkFile(const std::string & path) override;
     void removeDirectory(const std::string & path) override;
 
+    /// Hard links are simulated using server-side copying.
+    void createHardLink(const std::string & path_from, const std::string & path_to) override;
+
+    void moveFile(const std::string & path_from, const std::string & path_to) override;
+
     UnlinkMetadataFileOperationOutcomePtr unlinkMetadata(const std::string & path) override;
 
     void commit() override;