remove some useless code and sync some code for snapshot

Author: hanfei1991

src/Backups/BackupCoordinationOnCluster.cpp

@@ -102,7 +102,7 @@ namespace
     {
         BackupFileInfos file_infos;
 
-        static String serialize(const BackupFileInfos & file_infos_, bool is_lightweight_snapshot)
+        static String serialize(const BackupFileInfos & file_infos_)
         {
             WriteBufferFromOwnString out;
             writeBinary(file_infos_.size(), out);
@@ -115,17 +115,13 @@ namespace
                 writeBinary(info.base_checksum, out);
                 writeBinary(info.encrypted_by_disk, out);
                 writeBinary(info.reference_target, out);
-                if (is_lightweight_snapshot)
-                {
-                    writeBinary(info.object_key, out);
-                }
                 /// We don't store `info.data_file_name` and `info.data_file_index` because they're determined automalically
                 /// after reading file infos for all the hosts (see the class BackupCoordinationFileInfos).
             }
             return out.str();
         }
 
-        static FileInfos deserialize(const String & str, bool is_lightweight_snapshot)
+        static FileInfos deserialize(const String & str)
         {
             ReadBufferFromString in{str};
             FileInfos res;
@@ -142,10 +138,6 @@ namespace
                 readBinary(info.base_checksum, in);
                 readBinary(info.encrypted_by_disk, in);
                 readBinary(info.reference_target, in);
-                if (is_lightweight_snapshot)
-                {
-                    readBinary(info.object_key, in);
-                }
             }
             return res;
         }
@@ -172,7 +164,6 @@ size_t BackupCoordinationOnCluster::findCurrentHostIndex(const String & current_
 BackupCoordinationOnCluster::BackupCoordinationOnCluster(
     const UUID & backup_uuid_,
     bool is_plain_backup_,
-    bool is_lightweight_snapshot_,
     const String & root_zookeeper_path_,
     zkutil::GetZooKeeper get_zookeeper_,
     const BackupKeeperSettings & keeper_settings_,
@@ -191,7 +182,6 @@ BackupCoordinationOnCluster::BackupCoordinationOnCluster(
     , current_host(current_host_)
     , current_host_index(findCurrentHostIndex(current_host, all_hosts))
     , plain_backup(is_plain_backup_)
-    , lightweight_snapshot(is_lightweight_snapshot_)
     , process_list_element(process_list_element_)
     , log(getLogger("BackupCoordinationOnCluster"))
     , with_retries(log, get_zookeeper_, keeper_settings, process_list_element_, [root_zookeeper_path_](Coordination::ZooKeeperWithFaultInjection::Ptr zk) { zk->sync(root_zookeeper_path_); })
@@ -759,7 +749,7 @@ void BackupCoordinationOnCluster::addFileInfos(BackupFileInfos && file_infos_)
     }
 
     /// Serialize `file_infos_` and write it to ZooKeeper's nodes.
-    String file_infos_str = FileInfos::serialize(file_infos_, lightweight_snapshot);
+    String file_infos_str = FileInfos::serialize(file_infos_);
     serializeToMultipleZooKeeperNodes(zookeeper_path + "/file_infos/" + current_host, file_infos_str, "addFileInfos");
 }
 
@@ -798,7 +788,7 @@ void BackupCoordinationOnCluster::prepareFileInfos() const
     for (const String & host : hosts_with_file_infos)
     {
         String file_infos_str = deserializeFromMultipleZooKeeperNodes(zookeeper_path + "/file_infos/" + host, "prepareFileInfos");
-        auto deserialized_file_infos = FileInfos::deserialize(file_infos_str, lightweight_snapshot).file_infos;
+        auto deserialized_file_infos = FileInfos::deserialize(file_infos_str).file_infos;
         file_infos->addFileInfos(std::move(deserialized_file_infos), host);
     }
 }

src/Backups/BackupCoordinationOnCluster.h

@@ -24,7 +24,6 @@ class BackupCoordinationOnCluster : public IBackupCoordination
     BackupCoordinationOnCluster(
         const UUID & backup_uuid_,
         bool is_plain_backup_,
-        bool is_lightweight_snapshot_,
         const String & root_zookeeper_path_,
         zkutil::GetZooKeeper get_zookeeper_,
         const BackupKeeperSettings & keeper_settings_,
@@ -108,7 +107,6 @@ class BackupCoordinationOnCluster : public IBackupCoordination
     const String current_host;
     const size_t current_host_index;
     const bool plain_backup;
-    const bool lightweight_snapshot;
     const QueryStatusPtr process_list_element;
     const LoggerPtr log;
 

src/Backups/BackupImpl.cpp

@@ -92,23 +92,19 @@ BackupImpl::BackupImpl(
     BackupFactory::CreateParams params_,
     const ArchiveParams & archive_params_,
     std::shared_ptr<IBackupReader> reader_,
-    std::shared_ptr<IBackupReader> lightweight_snapshot_reader_)
+    SnapshotReaderCreator lightweight_snapshot_reader_creator_)
     : params(std::move(params_))
     , backup_info(params.backup_info)
     , backup_name_for_logging(backup_info.toStringForLogging())
     , use_archive(!archive_params_.archive_name.empty())
     , archive_params(archive_params_)
     , open_mode(OpenMode::READ)
     , reader(std::move(reader_))
-    , lightweight_snapshot_reader(std::move(lightweight_snapshot_reader_))
+    , lightweight_snapshot_reader_creator(lightweight_snapshot_reader_creator_)
     , version(INITIAL_BACKUP_VERSION)
     , base_backup_info(params.base_backup_info)
     , log(getLogger("BackupImpl"))
 {
-    if (params.is_lightweight_snapshot && !lightweight_snapshot_reader_)
-    {
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot create lightweight backup reader");
-    }
     open();
 }
 

src/Backups/BackupImpl.h

@@ -33,12 +33,14 @@ class BackupImpl : public IBackup
         size_t max_volume_size = 0;
     };
 
+    using SnapshotReaderCreator = std::function<std::shared_ptr<IBackupReader>(const String &, const String &)>;
+
     /// RESTORE
     BackupImpl(
         BackupFactory::CreateParams params_,
         const ArchiveParams & archive_params_,
         std::shared_ptr<IBackupReader> reader_,
-        std::shared_ptr<IBackupReader> lightweight_snapshot_reader_ = nullptr);
+        SnapshotReaderCreator lightweight_snapshot_reader_creator_ = {});
 
     /// BACKUP
     BackupImpl(
@@ -118,6 +120,7 @@ class BackupImpl : public IBackup
     /// Only used for lightweight backup, we read data from original object storage so the endpoint may be different from the backup files.
     std::shared_ptr<IBackupReader> lightweight_snapshot_reader;
     std::shared_ptr<IBackupWriter> lightweight_snapshot_writer;
+        SnapshotReaderCreator lightweight_snapshot_reader_creator;
     std::shared_ptr<IBackupCoordination> coordination;
 
     mutable std::mutex mutex;

src/Backups/BackupsWorker.cpp

@@ -1001,7 +1001,6 @@ BackupsWorker::makeBackupCoordination(bool on_cluster, const BackupSettings & ba
     return std::make_shared<BackupCoordinationOnCluster>(
         *backup_settings.backup_uuid,
         !backup_settings.deduplicate_files,
-        backup_settings.experimental_lightweight_snapshot,
         root_zk_path,
         get_zookeeper,
         keeper_settings,

src/Backups/IBackupEntry.h

@@ -46,6 +46,8 @@ class IBackupEntry
     virtual bool isFromRemoteFile() const { return false; }
     /// if it is a BackupEntryFromRemotePath, return the object key which the file refers to.
     virtual String getRemotePath() const { return "invalid remote path"; }
+    virtual String getEndpointURI() const { return "invalid endpoint"; }
+    virtual String getNamespace() const { return "invalid namespace"; }
     virtual String getFilePath() const { return ""; }
     virtual DiskPtr getDisk() const { return nullptr; }