Merge pull request ceph#64271 from benhanokh/full_dedup

rgw/dedup: Tech preview of full dedup support which means duplicate objects will be removed

Reviewed-by: Yuval Lifshitz <[email protected]>

Author: adamemerson

doc/radosgw/index.rst

@@ -91,4 +91,4 @@ Cluster with one API and then retrieve that data with the other API.
    Metrics <metrics>
    UADK Acceleration for Compression <uadk-accel>
    Bucket Logging <bucket_logging>
-
+   Full Object Deduplication <s3_objects_dedup>

doc/radosgw/s3_objects_dedup.rst

@@ -1,25 +1,28 @@
 =====================
 Full RGW Object Dedup
 =====================
-Adds a ``radosgw-admin`` command to collect and report deduplication stats.
-
-.. note:: This utility doesn't perform dedup and doesn't make any
-          change to the existing system and will only collect
-          statistics and report them.
+Adds ``radosgw-admin`` commands to remove duplicated RGW tail-objects and to collect and report deduplication stats.
 
 **************
 Admin commands
 **************
-- ``radosgw-admin dedup stats``:
-   Collects & displays last dedup statistics.
+- ``radosgw-admin dedup estimate``:
+   Starts a new dedup estimate session (aborting first existing session if exists).
+   
+   It doesn't make any change to the existing system and will only collect statistics and report them.
+- ``radosgw-admin dedup restart --yes-i-really-mean-it``:
+   Starts a new dedup session (aborting first existing session if exists).
+   It will perfrom a full dedup, finding duplicated tail-objects and removing them.
+
+  This command can lead to **data-loss** and should not be used on production data!!
 - ``radosgw-admin dedup pause``:
    Pauses an active dedup session (dedup resources are not released).
 - ``radosgw-admin dedup resume``:
    Resumes a paused dedup session.
 - ``radosgw-admin dedup abort``:
    Aborts an active dedup session and release all resources used by it.
-- ``radosgw-admin dedup estimate``:
-   Starts a new dedup estimate session (aborting first existing session if exists).
+- ``radosgw-admin dedup stats``:
+   Collects & displays last dedup statistics.
 
 ***************
 Skipped Objects
@@ -31,10 +34,7 @@ Dedup Estimate process skips the following objects:
 - Objects with different pools.
 - Objects with different storage classes.
 
-The dedup process itself (which will be released in a later Ceph release) will also skip
-**compressed** and **user-encrypted** objects, but the estimate
-process will accept them (since we don't have access to that
-information during the estimate process).
+The full dedup process skips all the above and it also skips **compressed** and **user-encrypted** objects.
 
 *******************
 Estimate Processing
@@ -54,6 +54,24 @@ the underlying media storing the objects (SSD/HDD) since the bucket indices are
 virtually always stored on a fast medium (SSD with heavy memory
 caching).
 
+*********************
+Full Dedup Processing
+*********************
+The Full Dedup process begins by constructing a dedup table from the bucket indices, similar to the estimate process above.
+
+This table is then scanned linearly to purge objects without duplicates, leaving only dedup candidates.
+
+Next, we iterate through these dedup candidate objects, reading their complete information from the object metadata (a per-object RADOS operation).
+During this step, we filter out **compressed** and **user-encrypted** objects.
+
+Following this, we calculate a strong-hash of the object data, which involves a full-object read and is a resource-intensive operation.
+This strong-hash ensures that the dedup candidates are indeed perfect matches.
+If they are, we proceed with the deduplication:
+
+- incrementing the reference count on the source tail-objects one by one.
+- copying the manifest from the source to the target.
+- removing all tail-objects on the target.
+
 ************
 Memory Usage
 ************

src/rgw/radosgw-admin/radosgw-admin.cc

@@ -156,7 +156,7 @@ void usage()
   cout << "  caps rm                          remove user capabilities\n";
   cout << "  dedup stats                      Display dedup statistics from the last run\n";
   cout << "  dedup estimate                   Runs dedup in estimate mode (no changes will be made)\n";
-  cout << "  dedup restart                    Restart dedup\n";
+  cout << "  dedup restart                    Restart dedup; must include --yes-i-really-mean-it to activate\n";
   cout << "  dedup abort                      Abort dedup\n";
   cout << "  dedup pause                      Pause dedup\n";
   cout << "  dedup resume                     Resume paused dedup\n";
@@ -9225,6 +9225,12 @@ int main(int argc, const char **argv)
 	dedup_type = dedup_req_type_t::DEDUP_TYPE_ESTIMATE;
       }
       else {
+	if (!yes_i_really_mean_it) {
+	  cerr << "Full Dedup is dangerous and could lead to data loss!\n"
+	       << "do you really mean it? (requires --yes-i-really-mean-it)"
+	       << std::endl;
+	  return EINVAL;
+	}
 	dedup_type = dedup_req_type_t::DEDUP_TYPE_FULL;
 #ifndef FULL_DEDUP_SUPPORT
 	std::cerr << "Only dedup estimate is supported!" << std::endl;

src/rgw/rgw_common.h

@@ -89,6 +89,7 @@ using ceph::crypto::MD5;
 #define RGW_ATTR_ETAG RGW_ATTR_PREFIX "etag"
 #define RGW_ATTR_CKSUM          RGW_ATTR_PREFIX "cksum"
 #define RGW_ATTR_SHA256         RGW_ATTR_PREFIX "x-amz-content-sha256"
+#define RGW_ATTR_BLAKE3         RGW_ATTR_PREFIX "blake3"
 #define RGW_ATTR_BUCKETS	RGW_ATTR_PREFIX "buckets"
 #define RGW_ATTR_META_PREFIX	RGW_ATTR_PREFIX RGW_AMZ_META_PREFIX
 #define RGW_ATTR_CONTENT_TYPE	RGW_ATTR_PREFIX "content_type"

src/rgw/rgw_dedup.cc

@@ -704,23 +704,23 @@ namespace rgw::dedup {
   //---------------------------------------------------------------------------
   static void init_cmp_pairs(const disk_record_t *p_rec,
                              const bufferlist    &etag_bl,
-                             bufferlist          &sha256_bl, // OUT PARAM
+                             bufferlist          &hash_bl, // OUT PARAM
                              librados::ObjectWriteOperation *p_op)
   {
     p_op->cmpxattr(RGW_ATTR_ETAG, CEPH_OSD_CMPXATTR_OP_EQ, etag_bl);
     // TBD: do we really need the secondary compare using the full manifest?
     // Can replace it with something cheaper like size/version?
     p_op->cmpxattr(RGW_ATTR_MANIFEST, CEPH_OSD_CMPXATTR_OP_EQ, p_rec->manifest_bl);
 
-    // SHA has 256 bit splitted into multiple 64bit units
+    // BLAKE3 hash has 256 bit splitted into multiple 64bit units
     const unsigned units = (256 / (sizeof(uint64_t)*8));
     static_assert(units == 4);
     for (unsigned i = 0; i < units; i++) {
-      ceph::encode(p_rec->s.sha256[i], sha256_bl);
+      ceph::encode(p_rec->s.hash[i], hash_bl);
     }
 
-    if (!p_rec->s.flags.sha256_calculated()) {
-      p_op->cmpxattr(RGW_ATTR_SHA256, CEPH_OSD_CMPXATTR_OP_EQ, sha256_bl);
+    if (!p_rec->s.flags.hash_calculated()) {
+      p_op->cmpxattr(RGW_ATTR_BLAKE3, CEPH_OSD_CMPXATTR_OP_EQ, hash_bl);
     }
   }
 
@@ -755,17 +755,17 @@ namespace rgw::dedup {
     ldpp_dout(dpp, 20) << __func__ << "::num_parts=" << p_tgt_rec->s.num_parts
                        << "::ETAG=" << etag_bl.to_str() << dendl;
 
-    bufferlist hash_bl, manifest_hash_bl, tgt_sha256_bl;
+    bufferlist hash_bl, manifest_hash_bl, tgt_hash_bl;
     crypto::digest<crypto::SHA1>(p_src_rec->manifest_bl).encode(hash_bl);
     // Use a shorter hash (64bit instead of 160bit)
     hash_bl.splice(0, 8, &manifest_hash_bl);
     librados::ObjectWriteOperation tgt_op;
-    init_cmp_pairs(p_tgt_rec, etag_bl, tgt_sha256_bl, &tgt_op);
+    init_cmp_pairs(p_tgt_rec, etag_bl, tgt_hash_bl, &tgt_op);
     tgt_op.setxattr(RGW_ATTR_SHARE_MANIFEST, manifest_hash_bl);
     tgt_op.setxattr(RGW_ATTR_MANIFEST, p_src_rec->manifest_bl);
-    if (p_tgt_rec->s.flags.sha256_calculated()) {
-      tgt_op.setxattr(RGW_ATTR_SHA256, tgt_sha256_bl);
-      p_stats->set_sha256_attrs++;
+    if (p_tgt_rec->s.flags.hash_calculated()) {
+      tgt_op.setxattr(RGW_ATTR_BLAKE3, tgt_hash_bl);
+      p_stats->set_hash_attrs++;
     }
 
     std::string src_oid, tgt_oid;
@@ -800,13 +800,13 @@ namespace rgw::dedup {
         // disk-record (as require an expensive random-disk-write).
         // When deduping C we can trust the shared_manifest state in the table and
         // skip a redundant update to SRC object attribute
-        bufferlist src_sha256_bl;
+        bufferlist src_hash_bl;
         librados::ObjectWriteOperation src_op;
-        init_cmp_pairs(p_src_rec, etag_bl, src_sha256_bl, &src_op);
+        init_cmp_pairs(p_src_rec, etag_bl, src_hash_bl, &src_op);
         src_op.setxattr(RGW_ATTR_SHARE_MANIFEST, manifest_hash_bl);
-        if (p_src_rec->s.flags.sha256_calculated()) {
-          src_op.setxattr(RGW_ATTR_SHA256, src_sha256_bl);
-          p_stats->set_sha256_attrs++;
+        if (p_src_rec->s.flags.hash_calculated()) {
+          src_op.setxattr(RGW_ATTR_BLAKE3, src_hash_bl);
+          p_stats->set_hash_attrs++;
         }
 
         ldpp_dout(dpp, 20) << __func__ <<"::send SRC CLS (Shared_Manifest)"<< dendl;
@@ -824,57 +824,49 @@ namespace rgw::dedup {
     return ret;
   }
 
-  using ceph::crypto::SHA256;
   //---------------------------------------------------------------------------
-  int Background::calc_object_sha256(const disk_record_t *p_rec, uint8_t *p_sha256)
+  int Background::calc_object_blake3(const disk_record_t *p_rec, uint8_t *p_hash)
   {
-    ldpp_dout(dpp, 20) << __func__ << "::p_rec->obj_name=" << p_rec->obj_name << dendl;
-    // Open questions -
-    // 1) do we need the secret if so what is the correct one to use?
-    // 2) are we passing the head/tail objects in the correct order?
+    ldpp_dout(dpp, 20) << __func__ << "::obj_name=" << p_rec->obj_name << dendl;
     RGWObjManifest manifest;
     try {
       auto bl_iter = p_rec->manifest_bl.cbegin();
       decode(manifest, bl_iter);
     } catch (buffer::error& err) {
-      ldpp_dout(dpp, 1)  << __func__ << "::ERROR: bad src manifest" << dendl;
+      ldpp_dout(dpp, 1)  << __func__ << "::ERROR: bad src manifest for: "
+                         << p_rec->obj_name << dendl;
       return -EINVAL;
     }
-    std::string oid;
-    build_oid(p_rec->bucket_id, p_rec->obj_name, &oid);
-    librados::IoCtx head_ioctx;
-    const char *secret = "0555b35654ad1656d804f1b017cd26e9";
-    TOPNSPC::crypto::HMACSHA256 hmac((const uint8_t*)secret, strlen(secret));
+
+    blake3_hasher hmac;
+    blake3_hasher_init(&hmac);
     for (auto p = manifest.obj_begin(dpp); p != manifest.obj_end(dpp); ++p) {
       rgw_raw_obj raw_obj = p.get_location().get_raw_obj(rados);
       rgw_rados_ref obj;
       int ret = rgw_get_rados_ref(dpp, rados_handle, raw_obj, &obj);
       if (ret < 0) {
-        ldpp_dout(dpp, 1) << __func__ << "::failed rgw_get_rados_ref() for raw_obj="
-                          << raw_obj << dendl;
+        ldpp_dout(dpp, 1) << __func__ << "::failed rgw_get_rados_ref() for oid: "
+                          << raw_obj.oid << ", err is " << cpp_strerror(-ret) << dendl;
         return ret;
       }
 
-      if (oid == raw_obj.oid) {
-        ldpp_dout(dpp, 20) << __func__ << "::manifest: head object=" << oid << dendl;
-        head_ioctx = obj.ioctx;
-      }
       bufferlist bl;
       librados::IoCtx ioctx = obj.ioctx;
       // read full object
       ret = ioctx.read(raw_obj.oid, bl, 0, 0);
       if (ret > 0) {
         for (const auto& bptr : bl.buffers()) {
-          hmac.Update((const unsigned char *)bptr.c_str(), bptr.length());
+          blake3_hasher_update(&hmac, (const unsigned char *)bptr.c_str(), bptr.length());
         }
       }
       else {
-        ldpp_dout(dpp, 1) << __func__ << "::ERR: failed to read " << oid
+        ldpp_dout(dpp, 1) << __func__ << "::ERR: failed to read " << raw_obj.oid
                           << ", error is " << cpp_strerror(-ret) << dendl;
         return ret;
       }
     }
-    hmac.Final(p_sha256);
+
+    blake3_hasher_finalize(&hmac, p_hash, BLAKE3_OUT_LEN);
     return 0;
   }
 
@@ -977,33 +969,33 @@ namespace rgw::dedup {
       memset(&p_rec->s.shared_manifest, 0, sizeof(p_rec->s.shared_manifest));
     }
 
-    itr = attrs.find(RGW_ATTR_SHA256);
+    itr = attrs.find(RGW_ATTR_BLAKE3);
     if (itr != attrs.end()) {
       try {
         auto bl_iter = itr->second.cbegin();
-        // SHA has 256 bit splitted into multiple 64bit units
+        // BLAKE3 hash 256 bit splitted into multiple 64bit units
         const unsigned units = (256 / (sizeof(uint64_t)*8));
         static_assert(units == 4);
         for (unsigned i = 0; i < units; i++) {
           uint64_t val;
           ceph::decode(val, bl_iter);
-          p_rec->s.sha256[i] = val;
+          p_rec->s.hash[i] = val;
         }
-        p_stats->valid_sha256_attrs++;
+        p_stats->valid_hash_attrs++;
         return 0;
       } catch (buffer::error& err) {
-        ldpp_dout(dpp, 1) << __func__ << "::ERR: failed SHA256 decode" << dendl;
+        ldpp_dout(dpp, 1) << __func__ << "::ERR: failed HASH decode" << dendl;
         return -EINVAL;
       }
     }
 
-    p_stats->invalid_sha256_attrs++;
+    p_stats->invalid_hash_attrs++;
     // TBD: redundant memset...
-    memset(p_rec->s.sha256, 0, sizeof(p_rec->s.sha256));
-    // CEPH_CRYPTO_HMACSHA256_DIGESTSIZE is 32 Bytes (32*8=256)
-    int ret = calc_object_sha256(p_rec, (uint8_t*)p_rec->s.sha256);
+    memset(p_rec->s.hash, 0, sizeof(p_rec->s.hash));
+    // BLAKE3_OUT_LEN is 32 Bytes
+    int ret = calc_object_blake3(p_rec, (uint8_t*)p_rec->s.hash);
     if (ret == 0) {
-      p_rec->s.flags.set_sha256_calculated();
+      p_rec->s.flags.set_hash_calculated();
     }
 
     return ret;
@@ -1177,18 +1169,18 @@ namespace rgw::dedup {
   }
 
   //---------------------------------------------------------------------------
-  static int write_sha256_object_attribute(const DoutPrefixProvider* const dpp,
+  static int write_blake3_object_attribute(const DoutPrefixProvider* const dpp,
                                            rgw::sal::Driver* driver,
                                            RGWRados* rados,
                                            const disk_record_t *p_rec)
   {
     bufferlist etag_bl;
-    bufferlist sha256_bl;
+    bufferlist hash_bl;
     librados::ObjectWriteOperation op;
     etag_to_bufferlist(p_rec->s.md5_high, p_rec->s.md5_low, p_rec->s.num_parts,
                        &etag_bl);
-    init_cmp_pairs(p_rec, etag_bl, sha256_bl /*OUT PARAM*/, &op);
-    op.setxattr(RGW_ATTR_SHA256, sha256_bl);
+    init_cmp_pairs(p_rec, etag_bl, hash_bl /*OUT PARAM*/, &op);
+    op.setxattr(RGW_ATTR_BLAKE3, hash_bl);
 
     std::string oid;
     librados::IoCtx ioctx;
@@ -1304,17 +1296,17 @@ namespace rgw::dedup {
       return 0;
     }
 
-    if (memcmp(src_rec.s.sha256, p_tgt_rec->s.sha256, sizeof(src_rec.s.sha256)) != 0) {
-      p_stats->sha256_mismatch++;
-      ldpp_dout(dpp, 10) << __func__ << "::SHA256 mismatch" << dendl;
-      // TBD: set sha256 attributes on head objects to save calc next time
-      if (src_rec.s.flags.sha256_calculated()) {
-        write_sha256_object_attribute(dpp, driver, rados, &src_rec);
-        p_stats->set_sha256_attrs++;
+    if (memcmp(src_rec.s.hash, p_tgt_rec->s.hash, sizeof(src_rec.s.hash)) != 0) {
+      p_stats->hash_mismatch++;
+      ldpp_dout(dpp, 10) << __func__ << "::HASH mismatch" << dendl;
+      // TBD: set hash attributes on head objects to save calc next time
+      if (src_rec.s.flags.hash_calculated()) {
+        write_blake3_object_attribute(dpp, driver, rados, &src_rec);
+        p_stats->set_hash_attrs++;
       }
-      if (p_tgt_rec->s.flags.sha256_calculated()) {
-        write_sha256_object_attribute(dpp, driver, rados, p_tgt_rec);
-        p_stats->set_sha256_attrs++;
+      if (p_tgt_rec->s.flags.hash_calculated()) {
+        write_blake3_object_attribute(dpp, driver, rados, p_tgt_rec);
+        p_stats->set_hash_attrs++;
       }
       return 0;
     }

src/rgw/rgw_dedup.h

@@ -179,7 +179,7 @@ namespace rgw::dedup {
                            remapper_t *remapper);
 
 #ifdef FULL_DEDUP_SUPPORT
-    int calc_object_sha256(const disk_record_t *p_rec, uint8_t *p_sha256);
+    int calc_object_blake3(const disk_record_t *p_rec, uint8_t *p_hash);
     int add_obj_attrs_to_record(rgw_bucket            *p_rb,
                                 disk_record_t         *p_rec,
                                 const rgw::sal::Attrs &attrs,