feat: support expire snapshots

Author: xiao.dong

src/iceberg/CMakeLists.txt

@@ -80,6 +80,7 @@ set(ICEBERG_SOURCES
     transform.cc
     transform_function.cc
     type.cc
+    update/expire_snapshots.cc
     update/pending_update.cc
     update/update_partition_spec.cc
     update/update_properties.cc

src/iceberg/meson.build

@@ -101,6 +101,7 @@ iceberg_sources = files(
     'transform.cc',
     'transform_function.cc',
     'type.cc',
+    'update/expire_snapshots.cc',
     'update/pending_update.cc',
     'update/update_partition_spec.cc',
     'update/update_properties.cc',

src/iceberg/table.cc

@@ -30,6 +30,7 @@
 #include "iceberg/table_properties.h"
 #include "iceberg/table_scan.h"
 #include "iceberg/transaction.h"
+#include "iceberg/update/expire_snapshots.h"
 #include "iceberg/update/update_partition_spec.h"
 #include "iceberg/update/update_properties.h"
 #include "iceberg/update/update_schema.h"
@@ -179,6 +180,13 @@ Result<std::shared_ptr<UpdateSchema>> Table::NewUpdateSchema() {
   return transaction->NewUpdateSchema();
 }
 
+Result<std::shared_ptr<ExpireSnapshots>> Table::NewExpireSnapshots() {
+  ICEBERG_ASSIGN_OR_RAISE(
+      auto transaction, Transaction::Make(shared_from_this(), Transaction::Kind::kUpdate,
+                                          /*auto_commit=*/true));
+  return transaction->NewExpireSnapshots();
+}
+
 Result<std::shared_ptr<StagedTable>> StagedTable::Make(
     TableIdentifier identifier, std::shared_ptr<TableMetadata> metadata,
     std::string metadata_location, std::shared_ptr<FileIO> io,

src/iceberg/table.h

@@ -144,6 +144,10 @@ class ICEBERG_EXPORT Table : public std::enable_shared_from_this<Table> {
   /// changes.
   virtual Result<std::shared_ptr<UpdateSchema>> NewUpdateSchema();
 
+  /// \brief Create a new ExpireSnapshots to remove expired snapshots and commit the
+  /// changes.
+  virtual Result<std::shared_ptr<ExpireSnapshots>> NewExpireSnapshots();
+
  protected:
   Table(TableIdentifier identifier, std::shared_ptr<TableMetadata> metadata,
         std::string metadata_location, std::shared_ptr<FileIO> io,

src/iceberg/table_metadata.cc

@@ -592,6 +592,12 @@ class TableMetadataBuilder::Impl {
   Result<int32_t> AddSchema(const Schema& schema, int32_t new_last_column_id);
   void SetLocation(std::string_view location);
 
+  Status SetRef(const std::string& name, std::shared_ptr<SnapshotRef> ref);
+  Status RemoveRef(const std::string& name);
+  Status AddSnapshot(std::shared_ptr<Snapshot> snapshot);
+  Status RemoveSnapshots(const std::vector<int64_t>& snapshot_ids);
+  Status RemovePartitionSpecs(const std::vector<int32_t>& spec_ids);
+
   Result<std::unique_ptr<TableMetadata>> Build();
 
  private:
@@ -1077,6 +1083,79 @@ int32_t TableMetadataBuilder::Impl::ReuseOrCreateNewSchemaId(
   return new_schema_id;
 }
 
+Status TableMetadataBuilder::Impl::SetRef(const std::string& name,
+                                          std::shared_ptr<SnapshotRef> ref) {
+  ICEBERG_PRECHECK(!metadata_.refs.contains(name),
+                   "Cannot set ref: {}, which is already exist.", name);
+  metadata_.refs[name] = ref;
+  if (ref->type() == SnapshotRefType::kBranch) {
+    auto retention = std::get<SnapshotRef::Branch>(ref->retention);
+    changes_.push_back(std::make_unique<table::SetSnapshotRef>(
+        name, ref->snapshot_id, ref->type(), retention.min_snapshots_to_keep,
+        retention.max_snapshot_age_ms, retention.max_ref_age_ms));
+  } else {
+    auto retention = std::get<SnapshotRef::Tag>(ref->retention);
+    changes_.push_back(std::make_unique<table::SetSnapshotRef>(
+        name, ref->snapshot_id, ref->type(), std::nullopt, std::nullopt,
+        retention.max_ref_age_ms));
+  }
+  return {};
+}
+
+Status TableMetadataBuilder::Impl::RemoveRef(const std::string& name) {
+  ICEBERG_PRECHECK(metadata_.refs.contains(name),
+                   "Cannot remove ref: {}, which is not exist.", name);
+
+  metadata_.refs.erase(name);
+  changes_.push_back(std::make_unique<table::RemoveSnapshotRef>(name));
+
+  return {};
+}
+
+Status TableMetadataBuilder::Impl::AddSnapshot(std::shared_ptr<Snapshot> snapshot) {
+  // TODO(xiao.dong) this is only for test, not official complete implementation
+  metadata_.snapshots.emplace_back(std::move(snapshot));
+  return {};
+}
+
+Status TableMetadataBuilder::Impl::RemoveSnapshots(
+    const std::vector<int64_t>& snapshot_ids) {
+  auto current_snapshot_id = metadata_.current_snapshot_id;
+  std::unordered_set<int64_t> snapshot_ids_set(snapshot_ids.begin(), snapshot_ids.end());
+  ICEBERG_PRECHECK(!snapshot_ids_set.contains(current_snapshot_id),
+                   "Cannot remove current snapshot: {}", current_snapshot_id);
+
+  if (!snapshot_ids.empty()) {
+    metadata_.snapshots =
+        metadata_.snapshots | std::views::filter([&](const auto& snapshot) {
+          return !snapshot_ids_set.contains(snapshot->snapshot_id);
+        }) |
+        std::ranges::to<std::vector<std::shared_ptr<iceberg::Snapshot>>>();
+    changes_.push_back(std::make_unique<table::RemoveSnapshots>(snapshot_ids));
+  }
+
+  return {};
+}
+
+Status TableMetadataBuilder::Impl::RemovePartitionSpecs(
+    const std::vector<int32_t>& spec_ids) {
+  auto default_spec_id = metadata_.default_spec_id;
+  std::unordered_set<int32_t> spec_ids_set(spec_ids.begin(), spec_ids.end());
+  ICEBERG_PRECHECK(!spec_ids_set.contains(default_spec_id),
+                   "Cannot remove default spec: {}", default_spec_id);
+
+  if (!spec_ids.empty()) {
+    metadata_.partition_specs =
+        metadata_.partition_specs | std::views::filter([&](const auto& spec) {
+          return !spec_ids_set.contains(spec->spec_id());
+        }) |
+        std::ranges::to<std::vector<std::shared_ptr<iceberg::PartitionSpec>>>();
+    changes_.push_back(std::make_unique<table::RemovePartitionSpecs>(spec_ids));
+  }
+
+  return {};
+}
+
 TableMetadataBuilder::TableMetadataBuilder(int8_t format_version)
     : impl_(std::make_unique<Impl>(format_version)) {}
 
@@ -1179,7 +1258,8 @@ TableMetadataBuilder& TableMetadataBuilder::AddPartitionSpec(
 
 TableMetadataBuilder& TableMetadataBuilder::RemovePartitionSpecs(
     const std::vector<int32_t>& spec_ids) {
-  throw IcebergError(std::format("{} not implemented", __FUNCTION__));
+  ICEBERG_BUILDER_RETURN_IF_ERROR(impl_->RemovePartitionSpecs(spec_ids));
+  return *this;
 }
 
 TableMetadataBuilder& TableMetadataBuilder::RemoveSchemas(
@@ -1207,7 +1287,8 @@ TableMetadataBuilder& TableMetadataBuilder::AddSortOrder(
 
 TableMetadataBuilder& TableMetadataBuilder::AddSnapshot(
     std::shared_ptr<Snapshot> snapshot) {
-  throw IcebergError(std::format("{} not implemented", __FUNCTION__));
+  ICEBERG_BUILDER_RETURN_IF_ERROR(impl_->AddSnapshot(std::move(snapshot)));
+  return *this;
 }
 
 TableMetadataBuilder& TableMetadataBuilder::SetBranchSnapshot(int64_t snapshot_id,
@@ -1217,11 +1298,13 @@ TableMetadataBuilder& TableMetadataBuilder::SetBranchSnapshot(int64_t snapshot_i
 
 TableMetadataBuilder& TableMetadataBuilder::SetRef(const std::string& name,
                                                    std::shared_ptr<SnapshotRef> ref) {
-  throw IcebergError(std::format("{} not implemented", __FUNCTION__));
+  ICEBERG_BUILDER_RETURN_IF_ERROR(impl_->SetRef(name, std::move(ref)));
+  return *this;
 }
 
 TableMetadataBuilder& TableMetadataBuilder::RemoveRef(const std::string& name) {
-  throw IcebergError(std::format("{} not implemented", __FUNCTION__));
+  ICEBERG_BUILDER_RETURN_IF_ERROR(impl_->RemoveRef(name));
+  return *this;
 }
 
 TableMetadataBuilder& TableMetadataBuilder::RemoveSnapshots(
@@ -1231,7 +1314,8 @@ TableMetadataBuilder& TableMetadataBuilder::RemoveSnapshots(
 
 TableMetadataBuilder& TableMetadataBuilder::RemoveSnapshots(
     const std::vector<int64_t>& snapshot_ids) {
-  throw IcebergError(std::format("{} not implemented", __FUNCTION__));
+  ICEBERG_BUILDER_RETURN_IF_ERROR(impl_->RemoveSnapshots(snapshot_ids));
+  return *this;
 }
 
 TableMetadataBuilder& TableMetadataBuilder::SuppressHistoricalSnapshots() {

src/iceberg/table_update.cc

@@ -92,7 +92,7 @@ void SetDefaultPartitionSpec::GenerateRequirements(TableUpdateContext& context)
 // RemovePartitionSpecs
 
 void RemovePartitionSpecs::ApplyTo(TableMetadataBuilder& builder) const {
-  throw IcebergError(std::format("{} not implemented", __FUNCTION__));
+  builder.RemovePartitionSpecs(spec_ids_);
 }
 
 void RemovePartitionSpecs::GenerateRequirements(TableUpdateContext& context) const {
@@ -143,7 +143,9 @@ void AddSnapshot::GenerateRequirements(TableUpdateContext& context) const {
 
 // RemoveSnapshots
 
-void RemoveSnapshots::ApplyTo(TableMetadataBuilder& builder) const {}
+void RemoveSnapshots::ApplyTo(TableMetadataBuilder& builder) const {
+  builder.RemoveSnapshots(snapshot_ids_);
+}
 
 void RemoveSnapshots::GenerateRequirements(TableUpdateContext& context) const {
   // RemoveSnapshots doesn't generate any requirements
@@ -152,7 +154,7 @@ void RemoveSnapshots::GenerateRequirements(TableUpdateContext& context) const {
 // RemoveSnapshotRef
 
 void RemoveSnapshotRef::ApplyTo(TableMetadataBuilder& builder) const {
-  throw IcebergError(std::format("{} not implemented", __FUNCTION__));
+  builder.RemoveRef(ref_name_);
 }
 
 void RemoveSnapshotRef::GenerateRequirements(TableUpdateContext& context) const {

src/iceberg/test/CMakeLists.txt

@@ -163,6 +163,7 @@ if(ICEBERG_BUILD_BUNDLE)
   add_iceberg_test(table_update_test
                    USE_BUNDLE
                    SOURCES
+                   expire_snapshots_test.cc
                    transaction_test.cc
                    update_partition_spec_test.cc
                    update_properties_test.cc

src/iceberg/test/expire_snapshots_test.cc

@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/update/expire_snapshots.h"
+
+#include "iceberg/test/matchers.h"
+#include "iceberg/test/update_test_base.h"
+
+namespace iceberg {
+
+class ExpireSnapshotsTest : public UpdateTestBase {
+ protected:
+};
+
+TEST_F(ExpireSnapshotsTest, Empty) {
+  ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewExpireSnapshots());
+  ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply());
+  EXPECT_THAT(result.snapshot_ids_to_remove.size(), 1);
+  EXPECT_THAT(result.snapshot_ids_to_remove.at(0), 3051729675574597004);
+  EXPECT_THAT(result.ref_to_remove.empty(), true);
+  EXPECT_THAT(result.schema_to_remove.empty(), true);
+  EXPECT_THAT(result.partition_spec_to_remove.empty(), true);
+}
+
+TEST_F(ExpireSnapshotsTest, Keep2) {
+  ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewExpireSnapshots());
+  update->RetainLast(2);
+  ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply());
+  EXPECT_THAT(result.snapshot_ids_to_remove.empty(), true);
+  EXPECT_THAT(result.ref_to_remove.empty(), true);
+  EXPECT_THAT(result.schema_to_remove.empty(), true);
+  EXPECT_THAT(result.partition_spec_to_remove.empty(), true);
+}
+
+TEST_F(ExpireSnapshotsTest, ExpireById) {
+  ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewExpireSnapshots());
+  update->ExpireSnapshotId(3051729675574597004);
+  update->RetainLast(2);
+  ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply());
+  EXPECT_THAT(result.snapshot_ids_to_remove.size(), 1);
+  EXPECT_THAT(result.snapshot_ids_to_remove.at(0), 3051729675574597004);
+  EXPECT_THAT(result.ref_to_remove.empty(), true);
+  EXPECT_THAT(result.schema_to_remove.empty(), true);
+  EXPECT_THAT(result.partition_spec_to_remove.empty(), true);
+}
+
+TEST_F(ExpireSnapshotsTest, ExpireByIdNotExist) {
+  ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewExpireSnapshots());
+  update->ExpireSnapshotId(3055729675574597003);
+  update->RetainLast(2);
+  auto result = update->Apply();
+  EXPECT_THAT(result.has_value(), false);
+  EXPECT_THAT(result.error().message.contains("Snapshot:3055729675574597003 not exist"),
+              true);
+}
+
+TEST_F(ExpireSnapshotsTest, ExpireOlderThan1) {
+  ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewExpireSnapshots());
+  update->ExpireOlderThan(1515100955770 - 1);
+  ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply());
+  EXPECT_THAT(result.snapshot_ids_to_remove.empty(), true);
+  EXPECT_THAT(result.ref_to_remove.empty(), true);
+  EXPECT_THAT(result.schema_to_remove.empty(), true);
+  EXPECT_THAT(result.partition_spec_to_remove.empty(), true);
+}
+
+TEST_F(ExpireSnapshotsTest, ExpireOlderThan2) {
+  ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewExpireSnapshots());
+  update->ExpireOlderThan(1515100955770 + 1);
+  ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply());
+  EXPECT_THAT(result.snapshot_ids_to_remove.size(), 1);
+  EXPECT_THAT(result.snapshot_ids_to_remove.at(0), 3051729675574597004);
+  EXPECT_THAT(result.ref_to_remove.empty(), true);
+  EXPECT_THAT(result.schema_to_remove.empty(), true);
+  EXPECT_THAT(result.partition_spec_to_remove.empty(), true);
+}
+
+}  // namespace iceberg