rgw/dedup: full object dedup

Design Document:
https://docs.google.com/document/d/152VyCTR2NlZ6ongbe6-CJfP4qxr1_zH83FB_WukWD7c

Signed-off-by: Gabriel BenHanokh <[email protected]>

Author: benhanokh

doc/radosgw/s3_objects_dedup.rst

@@ -0,0 +1,79 @@
+======================
+Full RGW Object Dedup:
+======================
+Add a radosgw-admin command to collect and report deduplication stats
+
+.. note:: This utility doesn’t perform dedup and doesn’t make any
+          change to the existing system and will only collect
+          statistics and report them.
+
+----
+
+***************
+Admin commands:
+***************
+- ``radosgw-admin dedup stats``:
+   Collects & displays last dedup statistics
+- ``radosgw-admin dedup pause``:
+   Pauses active dedup session (dedup resources are not released)
+- ``radosgw-admin dedup resume``:
+   Resumes a paused dedup session
+- ``radosgw-admin dedup abort``:
+   Aborts active dedup session and release all resources used by it
+- ``radosgw-admin dedup estimate``
+    Starts a new dedup estimate session (aborting first existing session if exists)
+
+----
+
+****************
+Skipped Objects:
+****************
+Dedup Estimates skips the following objects:
+
+- Objects smaller than 4MB (unless they are multipart)
+- Objects with different placement rules
+- Objects with different pools
+- Objects with different same storage-classes
+
+The Dedup process itself (which will be released later) will also skip
+**compressed** and **user-encrypted** objects, but the estimate
+process will accept them (since we don't have access to that
+information during the estimate process)
+
+----
+
+********************
+Estimate Processing:
+********************
+The Dedup Estimate process collects all the needed information directly from
+the bucket-indices reading one full bucket-index object with 1000's of
+entries at a time.
+
+The Bucket-Indices objects are sharded between the participating
+members so every bucket-index object is read exactly one time.
+The sharding allow processing to scale almost linearly spliting the
+load evenly between the participating members.
+
+The Dedup Estimate process does not access the objects themselves
+(data/metadata) which means its processing time won't be affected by
+the underlined media storing the objects (SSD/HDD) since the bucket-indices are
+virtually always stored on a fast medium (SSD with heavy memory
+caching)
+
+----
+
+*************
+Memory Usage:
+*************
+ +---------------++-----------+
+ | RGW Obj Count |  Memory    |
+ +===============++===========+
+ | | ____1M      | | ___8MB   |
+ | | ____4M      | | __16MB   |
+ | | ___16M      | | __32MB   |
+ | | ___64M      | | __64MB   |
+ | | __256M      | | _128MB   |
+ | | _1024M( 1G) | | _256MB   |
+ | | _4096M( 4G) | | _512MB   |
+ | | 16384M(16G) | | 1024MB   |
+ +---------------+------------+

src/common/subsys.h

@@ -104,6 +104,7 @@ SUBSYS(seastore_device, 0, 5)
 SUBSYS(seastore_backref, 0, 5)
 SUBSYS(alienstore, 0, 5)
 SUBSYS(mclock, 1, 5)
+SUBSYS(rgw_dedup, 1, 5)
 SUBSYS(cyanstore, 0, 5)
 SUBSYS(ceph_exporter, 1, 5)
 SUBSYS(memstore, 1, 5)

src/rgw/CMakeLists.txt

@@ -159,6 +159,11 @@ set(librgw_common_srcs
   rgw_bucket_encryption.cc
   rgw_tracer.cc
   rgw_lua_background.cc
+  rgw_dedup.cc
+  rgw_dedup_table.cc
+  rgw_dedup_store.cc
+  rgw_dedup_utils.cc
+  rgw_dedup_cluster.cc
   rgw_data_access.cc
   rgw_realm_watcher.cc
   driver/rados/account.cc

src/rgw/radosgw-admin/radosgw-admin.cc

@@ -48,7 +48,6 @@ extern "C" {
 
 #include "radosgw-admin/orphan.h"
 #include "radosgw-admin/sync_checkpoint.h"
-
 #include "rgw_user.h"
 #include "rgw_otp.h"
 #include "rgw_rados.h"
@@ -78,7 +77,7 @@ extern "C" {
 #include "rgw_data_access.h"
 #include "rgw_account.h"
 #include "rgw_bucket_logging.h"
-
+#include "rgw_dedup_cluster.h"
 #include "services/svc_sync_modules.h"
 #include "services/svc_cls.h"
 #include "services/svc_bilog_rados.h"
@@ -151,6 +150,12 @@ void usage()
   cout << "  user policy list attached        list attached managed policies\n";
   cout << "  caps add                         add user capabilities\n";
   cout << "  caps rm                          remove user capabilities\n";
+  cout << "  dedup stats                      Display dedup statistics from the last run\n";
+  cout << "  dedup estimate                   Runs dedup in estimate mode (no changes will be made)\n";
+  cout << "  dedup restart                    Restart dedup\n";
+  cout << "  dedup abort                      Abort dedup\n";
+  cout << "  dedup pause                      Pause dedup\n";
+  cout << "  dedup resume                     Resume paused dedup\n";
   cout << "  subuser create                   create a new subuser\n" ;
   cout << "  subuser modify                   modify subuser\n";
   cout << "  subuser rm                       remove subuser\n";
@@ -742,6 +747,12 @@ enum class OPT {
   QUOTA_SET,
   QUOTA_ENABLE,
   QUOTA_DISABLE,
+  DEDUP_STATS,
+  DEDUP_ESTIMATE,
+  DEDUP_ABORT,
+  DEDUP_RESTART,
+  DEDUP_PAUSE,
+  DEDUP_RESUME,
   GC_LIST,
   GC_PROCESS,
   LC_LIST,
@@ -989,6 +1000,12 @@ static SimpleCmd::Commands all_cmds = {
   { "ratelimit set", OPT::RATELIMIT_SET },
   { "ratelimit enable", OPT::RATELIMIT_ENABLE },
   { "ratelimit disable", OPT::RATELIMIT_DISABLE },
+  { "dedup stats", OPT::DEDUP_STATS },
+  { "dedup estimate", OPT::DEDUP_ESTIMATE },
+  { "dedup abort", OPT::DEDUP_ABORT },
+  { "dedup restart", OPT::DEDUP_RESTART },
+  { "dedup pause", OPT::DEDUP_PAUSE },
+  { "dedup resume", OPT::DEDUP_RESUME },
   { "gc list", OPT::GC_LIST },
   { "gc process", OPT::GC_PROCESS },
   { "lc list", OPT::LC_LIST },
@@ -4509,6 +4526,12 @@ int main(int argc, const char **argv)
 			 OPT::BI_LIST,
 			 OPT::OLH_GET,
 			 OPT::OLH_READLOG,
+			 OPT::DEDUP_STATS,
+			 OPT::DEDUP_ESTIMATE,
+			 OPT::DEDUP_ABORT,     // TBD - not READ-ONLY
+			 OPT::DEDUP_RESTART,   // TBD - not READ-ONLY
+			 OPT::DEDUP_PAUSE,
+			 OPT::DEDUP_RESUME,
 			 OPT::GC_LIST,
 			 OPT::LC_LIST,
 			 OPT::ORPHANS_LIST_JOBS,
@@ -9168,6 +9191,71 @@ int main(int argc, const char **argv)
     }
   }
 
+  if (opt_cmd == OPT::DEDUP_STATS    ||
+      opt_cmd == OPT::DEDUP_ESTIMATE ||
+      opt_cmd == OPT::DEDUP_ABORT    ||
+      opt_cmd == OPT::DEDUP_PAUSE    ||
+      opt_cmd == OPT::DEDUP_RESUME   ||
+      opt_cmd == OPT::DEDUP_RESTART) {
+
+    using namespace rgw::dedup;
+    rgw::sal::RadosStore *store = dynamic_cast<rgw::sal::RadosStore*>(driver);
+    if (!store) {
+      cerr << "ERROR: this command can only work when the cluster has a RADOS "
+	   << "backing store." << std::endl;
+      return EPERM;
+    }
+
+    if (opt_cmd == OPT::DEDUP_STATS) {
+      int ret = cluster::collect_all_shard_stats(store, formatter.get(), dpp());
+      if (ret == 0) {
+	formatter->flush(cout);
+      }
+      else {
+	cerr << "ERROR: Failed reading stat counters" << std::endl;
+      }
+      return ret;
+    }
+
+    if (opt_cmd == OPT::DEDUP_ABORT || opt_cmd == OPT::DEDUP_PAUSE || opt_cmd == OPT::DEDUP_RESUME) {
+      urgent_msg_t urgent_msg;
+      if (opt_cmd == OPT::DEDUP_ABORT) {
+	urgent_msg = URGENT_MSG_ABORT;
+      }
+      else if (opt_cmd == OPT::DEDUP_PAUSE) {
+	urgent_msg = URGENT_MSG_PASUE;
+      }
+      else {
+	urgent_msg = URGENT_MSG_RESUME;
+      }
+      return cluster::dedup_control(store, dpp(), urgent_msg);
+    }
+
+    if (opt_cmd == OPT::DEDUP_RESTART || opt_cmd == OPT::DEDUP_ESTIMATE) {
+      dedup_req_type_t dedup_type = dedup_req_type_t::DEDUP_TYPE_NONE;
+      if (opt_cmd == OPT::DEDUP_ESTIMATE) {
+	dedup_type = dedup_req_type_t::DEDUP_TYPE_ESTIMATE;
+      }
+      else {
+	dedup_type = dedup_req_type_t::DEDUP_TYPE_FULL;
+#ifndef FULL_DEDUP_SUPPORT
+	std::cerr << "Only dedup estimate is supported!" << std::endl;
+	return EPERM;
+#endif
+      }
+
+      int ret = cluster::dedup_restart_scan(store, dedup_type, dpp());
+      if (ret == 0) {
+	std::cout << "Dedup was restarted successfully" << std::endl;
+      }
+      else {
+	std::cerr << "Dedup failed to restart" << std::endl;
+	std::cerr << "Error is: " << ret << "::" << cpp_strerror(ret) << std::endl;
+      }
+      return ret;
+    }
+  }
+
   if (opt_cmd == OPT::GC_LIST) {
     int index = 0;
     bool truncated;

src/rgw/rgw_appmain.cc

@@ -66,6 +66,7 @@
 #include "rgw_asio_frontend.h"
 #include "rgw_dmclock_scheduler_ctx.h"
 #include "rgw_lua.h"
+#include "rgw_dedup.h"
 #ifdef WITH_RADOSGW_DBSTORE
 #include "rgw_sal_dbstore.h"
 #endif
@@ -534,7 +535,9 @@ int rgw::AppMain::init_frontends2(RGWLib* rgwlib)
     if (env.lua.background) {
       rgw_pauser->add_pauser(env.lua.background);
     }
-
+    if (dedup_background) {
+      rgw_pauser->add_pauser(dedup_background.get());
+    }
     need_context_pool();
     reloader = std::make_unique<RGWRealmReloader>(
         env, *implicit_tenant_context, service_map_meta, rgw_pauser.get(), *context_pool);
@@ -558,7 +561,6 @@ void rgw::AppMain::init_lua()
   rgw::sal::Driver* driver = env.driver;
   int r{0};
   std::string install_dir;
-
 #ifdef WITH_RADOSGW_LUA_PACKAGES
   rgw::lua::packages_t failed_packages;
   r = rgw::lua::install_packages(dpp, driver, null_yield, g_conf().get_val<std::string>("rgw_luarocks_location"),
@@ -583,6 +585,21 @@ void rgw::AppMain::init_lua()
   }
 } /* init_lua */
 
+void rgw::AppMain::init_dedup()
+{
+  rgw::sal::Driver* driver = env.driver;
+  if (driver->get_name() == "rados") { /* Supported for only RadosStore */
+    try {
+      dedup_background = std::make_unique<rgw::dedup::Background>(driver, dpp->get_cct());
+      dedup_background->start();
+      dedup_background->watch_reload(dpp);
+    }
+    catch (const std::runtime_error&) {
+      ldpp_dout(dpp, 0) << __func__ << "::failed create dedup background job" << dendl;
+    }
+  }
+}
+
 void rgw::AppMain::shutdown(std::function<void(void)> finalize_async_signals)
 {
   // stop the realm reloader
@@ -596,6 +613,10 @@ void rgw::AppMain::shutdown(std::function<void(void)> finalize_async_signals)
     if (g_conf().get_val<bool>("rgw_lua_enable"))
       static_cast<rgw::sal::RadosLuaManager*>(env.lua.manager.get())->
           unwatch_reload(dpp);
+
+    if (dedup_background) {
+      dedup_background->unwatch_reload(dpp);
+    }
   }
 
   for (auto& fe : fes) {
@@ -605,6 +626,10 @@ void rgw::AppMain::shutdown(std::function<void(void)> finalize_async_signals)
   ldh.reset(nullptr); // deletes ldap helper if it was created
   rgw_log_usage_finalize();
 
+  if (dedup_background) {
+    dedup_background->shutdown();
+  }
+
   if (lua_background) {
     lua_background->shutdown();
   }

src/rgw/rgw_common.h

@@ -87,7 +87,8 @@ using ceph::crypto::MD5;
 #define RGW_ATTR_LC		RGW_ATTR_PREFIX "lc"
 #define RGW_ATTR_CORS		RGW_ATTR_PREFIX "cors"
 #define RGW_ATTR_ETAG RGW_ATTR_PREFIX "etag"
-#define RGW_ATTR_CKSUM    	RGW_ATTR_PREFIX "cksum"
+#define RGW_ATTR_CKSUM          RGW_ATTR_PREFIX "cksum"
+#define RGW_ATTR_SHA256         RGW_ATTR_PREFIX "x-amz-content-sha256"
 #define RGW_ATTR_BUCKETS	RGW_ATTR_PREFIX "buckets"
 #define RGW_ATTR_META_PREFIX	RGW_ATTR_PREFIX RGW_AMZ_META_PREFIX
 #define RGW_ATTR_CONTENT_TYPE	RGW_ATTR_PREFIX "content_type"
@@ -102,6 +103,7 @@ using ceph::crypto::MD5;
 #define RGW_ATTR_SHADOW_OBJ    	RGW_ATTR_PREFIX "shadow_name"
 #define RGW_ATTR_MANIFEST    	RGW_ATTR_PREFIX "manifest"
 #define RGW_ATTR_USER_MANIFEST  RGW_ATTR_PREFIX "user_manifest"
+#define RGW_ATTR_SHARE_MANIFEST RGW_ATTR_PREFIX "shared_manifest"
 #define RGW_ATTR_AMZ_WEBSITE_REDIRECT_LOCATION	RGW_ATTR_PREFIX RGW_AMZ_WEBSITE_REDIRECT_LOCATION
 #define RGW_ATTR_SLO_MANIFEST   RGW_ATTR_PREFIX "slo_manifest"
 /* Information whether an object is SLO or not must be exposed to