Merge pull request ceph#55076 from linuxbox2/wip-rgw-cksum

rgw: implement S3 additional checksum support

Author: mattbenjamin

.gitmodules

@@ -75,3 +75,10 @@
 [submodule "src/qatzip"]
 	path = src/qatzip
 	url = https://github.com/intel/qatzip.git
+[submodule "src/boost_redis"]
+	path = src/boost_redis
+	url = https://github.com/boostorg/redis.git
+[submodule "src/BLAKE3"]
+	path = src/BLAKE3
+	url = https://github.com/BLAKE3-team/BLAKE3.git
+

src/BLAKE3

@@ -196,6 +196,9 @@ if(HAS_GLIBCXX_ASSERTIONS AND CMAKE_BUILD_TYPE STREQUAL Debug)
   add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-D_GLIBCXX_ASSERTIONS>)
 endif()
 
+# add BLAKE3 before we clobber CMAKE_ASM_COMPILER
+add_subdirectory(BLAKE3/c EXCLUDE_FROM_ALL)
+
 include(SIMDExt)
 if(HAVE_INTEL)
   if(APPLE)

src/CMakeLists.txt

@@ -91,7 +91,6 @@ namespace TOPNSPC::crypto {
         SHA512 () : OpenSSLDigest(EVP_sha512()) { }
     };
 
-
 # if OPENSSL_VERSION_NUMBER < 0x10100000L
   class HMAC {
   private:

src/common/ceph_crypto.h

@@ -70,6 +70,7 @@ set(librgw_common_srcs
   rgw_bucket.cc
   rgw_bucket_layout.cc
   rgw_cache.cc
+  rgw_cksum_pipe.cc
   rgw_common.cc
   rgw_compression.cc
   rgw_cors.cc
@@ -304,7 +305,8 @@ target_link_libraries(rgw_common
     RapidJSON::RapidJSON
     Boost::context
     ${FMT_LIB}
-    OpenSSL::SSL)
+    OpenSSL::SSL
+    BLAKE3::blake3)
 target_include_directories(rgw_common
   PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw/services"
   PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw"

src/rgw/CMakeLists.txt

@@ -811,6 +811,7 @@ int D4NFilterWriter::process(bufferlist&& data, uint64_t offset)
 int D4NFilterWriter::complete(size_t accounted_size, const std::string& etag,
                        ceph::real_time *mtime, ceph::real_time set_mtime,
                        std::map<std::string, bufferlist>& attrs,
+		       const std::optional<rgw::cksum::Cksum>& cksum,
                        ceph::real_time delete_at,
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
@@ -830,7 +831,7 @@ int D4NFilterWriter::complete(size_t accounted_size, const std::string& etag,
     ldpp_dout(save_dpp, 10) << "D4NFilterWriter::" << __func__ << "(): ObjectDirectory set method failed." << dendl;
 
   /* Retrieve complete set of attrs */
-  int ret = next->complete(accounted_size, etag, mtime, set_mtime, attrs,
+  int ret = next->complete(accounted_size, etag, mtime, set_mtime, attrs, cksum,
 			delete_at, if_match, if_nomatch, user_data, zones_trace,
 			canceled, rctx, flags);
   obj->get_obj_attrs(rctx.y, save_dpp, NULL);

src/rgw/driver/d4n/rgw_sal_d4n.cc

@@ -219,14 +219,15 @@ class D4NFilterWriter : public FilterWriter {
     virtual int prepare(optional_yield y);
     virtual int process(bufferlist&& data, uint64_t offset) override;
     virtual int complete(size_t accounted_size, const std::string& etag,
-                       ceph::real_time *mtime, ceph::real_time set_mtime,
-                       std::map<std::string, bufferlist>& attrs,
-                       ceph::real_time delete_at,
-                       const char *if_match, const char *if_nomatch,
-                       const std::string *user_data,
-                       rgw_zone_set *zones_trace, bool *canceled,
-                       const req_context& rctx,
-                       uint32_t flags) override;
+			 ceph::real_time *mtime, ceph::real_time set_mtime,
+			 std::map<std::string, bufferlist>& attrs,
+			 const std::optional<rgw::cksum::Cksum>& cksum,
+			 ceph::real_time delete_at,
+			 const char *if_match, const char *if_nomatch,
+			 const std::string *user_data,
+			 rgw_zone_set *zones_trace, bool *canceled,
+			 const req_context& rctx,
+			 uint32_t flags) override;
    bool is_atomic() { return atomic; };
    const DoutPrefixProvider* dpp() { return save_dpp; } 
 };

src/rgw/driver/d4n/rgw_sal_d4n.h

@@ -1596,6 +1596,7 @@ int DaosMultipartUpload::init(const DoutPrefixProvider* dpp, optional_yield y,
 
   multipart_upload_info upload_info;
   upload_info.dest_placement = dest_placement;
+  upload_info.cksum_type = cksum_type;
 
   ent.encode(bl);
   encode(attrs, bl);
@@ -1968,6 +1969,7 @@ int DaosMultipartUpload::get_info(const DoutPrefixProvider* dpp,
 
   // Now decode the placement rule
   decode(upload_info, iter);
+  cksum_type = upload_info.cksum_type;
   placement = upload_info.dest_placement;
   *rule = &placement;
 

src/rgw/driver/daos/rgw_sal_daos.cc

@@ -742,6 +742,7 @@ class DaosAtomicWriter : public StoreWriter {
   virtual int complete(size_t accounted_size, const std::string& etag,
                        ceph::real_time* mtime, ceph::real_time set_mtime,
                        std::map<std::string, bufferlist>& attrs,
+		       const std::optional<rgw::cksum::Cksum>& cksum,
                        ceph::real_time delete_at, const char* if_match,
                        const char* if_nomatch, const std::string* user_data,
                        rgw_zone_set* zones_trace, bool* canceled,
@@ -788,6 +789,7 @@ class DaosMultipartWriter : public StoreWriter {
   virtual int complete(size_t accounted_size, const std::string& etag,
                        ceph::real_time* mtime, ceph::real_time set_mtime,
                        std::map<std::string, bufferlist>& attrs,
+		       const std::optional<rgw::cksum::Cksum>& cksum,
                        ceph::real_time delete_at, const char* if_match,
                        const char* if_nomatch, const std::string* user_data,
                        rgw_zone_set* zones_trace, bool* canceled,
@@ -810,6 +812,10 @@ class DaosMultipartPart : public StoreMultipartPart {
   virtual const std::string& get_etag() { return info.etag; }
   virtual ceph::real_time& get_mtime() { return info.modified; }
 
+  virtual const std::optional<rgw::cksum::Cksum>& get_cksum() {
+    return info.cksum;
+  }
+
   friend class DaosMultipartUpload;
 };
 

src/rgw/driver/daos/rgw_sal_daos.h

@@ -791,6 +791,7 @@ class MotrAtomicWriter : public StoreWriter {
   virtual int complete(size_t accounted_size, const std::string& etag,
                        ceph::real_time *mtime, ceph::real_time set_mtime,
                        std::map<std::string, bufferlist>& attrs,
+		       const std::optional<rgw::cksum::Cksum>& cksum,
                        ceph::real_time delete_at,
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
@@ -839,6 +840,7 @@ class MotrMultipartWriter : public StoreWriter {
   virtual int complete(size_t accounted_size, const std::string& etag,
                        ceph::real_time *mtime, ceph::real_time set_mtime,
                        std::map<std::string, bufferlist>& attrs,
+		       const std::optional<rgw::cksum::Cksum>& cksum,
                        ceph::real_time delete_at,
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
@@ -888,6 +890,9 @@ class MotrMultipartPart : public StoreMultipartPart {
   virtual uint64_t get_size() { return info.accounted_size; }
   virtual const std::string& get_etag() { return info.etag; }
   virtual ceph::real_time& get_mtime() { return info.modified; }
+  virtual const std::optional<rgw::cksum::Cksum>& get_cksum() {
+    return info.cksum;
+  }
 
   RGWObjManifest& get_manifest() { return info.manifest; }