rgw_cksum: permit fallback to checksum-type from create-multipart, in upload-part

There appear to be workloads that provide a checksum algorithm in
create-multipart-upload, but do not provide (what must be) the
corresponding algorithm when uploading the parts.  (complete-multipart-upload
has no checksum argument, the value is implicit.)

This behavior is inconsistent with at least some SDKs, but it is
possibly accepted behavior in AWS S3, and is not logically necessary,
since the originally supplied checksum type is already known.

Therefore, change the behavior of upload-part to fall back to a
checksum type that was sent with the corresponding create-multipart-upload
request, if one is present, rather than failing with InvalidRequest.

Fixes: https://tracker.ceph.com/issues/68513

Signed-off-by: Matt Benjamin <[email protected]>
(cherry picked from commit 6b487a4)

Author: mattbenjamin

src/rgw/rgw_cksum_pipe.cc

@@ -18,6 +18,7 @@
 #include <string>
 #include <fmt/format.h>
 #include <boost/algorithm/string.hpp>
+#include "rgw_cksum.h"
 #include "rgw_common.h"
 #include "common/dout.h"
 #include "rgw_client_io.h"
@@ -34,7 +35,8 @@ namespace rgw::putobj {
   {}
 
   std::unique_ptr<RGWPutObj_Cksum> RGWPutObj_Cksum::Factory(
-    rgw::sal::DataProcessor* next, const RGWEnv& env)
+    rgw::sal::DataProcessor* next, const RGWEnv& env,
+    rgw::cksum::Type override_type)
   {
     /* look for matching headers */
     auto algo_header = cksum_algorithm_hdr(env);
@@ -49,6 +51,13 @@ namespace rgw::putobj {
       throw rgw::io::Exception(EINVAL, std::system_category());
     }
     /* no checksum header */
+    if (override_type != rgw::cksum::Type::none) {
+      /* XXXX safe? do we need to fixup env as well? */
+      auto algo_header = cksum_algorithm_hdr(override_type);
+      return
+	std::make_unique<RGWPutObj_Cksum>(
+			   next, override_type, std::move(algo_header));
+    }
     return std::unique_ptr<RGWPutObj_Cksum>();
   }
 

src/rgw/rgw_cksum_pipe.h

@@ -20,6 +20,7 @@
 #include <tuple>
 #include <cstring>
 #include <boost/algorithm/string/case_conv.hpp>
+#include "rgw_cksum.h"
 #include "rgw_cksum_digest.h"
 #include "rgw_common.h"
 #include "rgw_putobj.h"
@@ -29,6 +30,38 @@ namespace rgw::putobj {
   namespace cksum = rgw::cksum;
   using cksum_hdr_t = std::pair<const char*, const char*>;
 
+  static inline const cksum_hdr_t cksum_algorithm_hdr(rgw::cksum::Type t) {
+    static constexpr std::string_view hdr =
+      "HTTP_X_AMZ_SDK_CHECKSUM_ALGORITHM";
+    using rgw::cksum::Type;
+    switch (t) {
+    case Type::sha256:
+      return cksum_hdr_t(hdr.data(), "SHA256");
+      break;
+    case Type::crc32:
+      return cksum_hdr_t(hdr.data(), "CRC32");
+      break;
+    case Type::crc32c:
+      return cksum_hdr_t(hdr.data(), "CRC32C");
+      break;
+    case Type::xxh3:
+      return cksum_hdr_t(hdr.data(), "XX3");
+      break;
+    case Type::sha1:
+      return cksum_hdr_t(hdr.data(), "SHA1");
+      break;
+    case Type::sha512:
+      return cksum_hdr_t(hdr.data(), "SHA512");
+      break;
+    case Type::blake3:
+      return cksum_hdr_t(hdr.data(), "BLAKE3");
+      break;
+    default:
+      break;
+    };
+    return cksum_hdr_t(nullptr, nullptr);;
+  }
+
   static inline const cksum_hdr_t cksum_algorithm_hdr(const RGWEnv& env) {
     /* If the individual checksum value you provide through
        x-amz-checksum-algorithm doesn't match the checksum algorithm
@@ -102,7 +135,8 @@ namespace rgw::putobj {
     using VerifyResult = std::tuple<bool, const cksum::Cksum&>;
 
     static std::unique_ptr<RGWPutObj_Cksum> Factory(
-      rgw::sal::DataProcessor* next, const RGWEnv&);
+      rgw::sal::DataProcessor* next, const RGWEnv&,
+      rgw::cksum::Type override_type);
 
     RGWPutObj_Cksum(rgw::sal::DataProcessor* next, rgw::cksum::Type _type,
 		    cksum_hdr_t&& _hdr);

src/rgw/rgw_op.cc

@@ -25,6 +25,7 @@
 #include "common/ceph_json.h"
 #include "common/static_ptr.h"
 #include "common/perf_counters_key.h"
+#include "rgw_cksum.h"
 #include "rgw_cksum_digest.h"
 #include "rgw_common.h"
 #include "common/split.h"
@@ -4333,6 +4334,9 @@ void RGWPutObj::execute(optional_yield y)
       }
       return;
     }
+
+    multipart_cksum_type = upload->cksum_type;
+
     /* upload will go out of scope, so copy the dest placement for later use */
     s->dest_placement = *pdest_placement;
     pdest_placement = &s->dest_placement;
@@ -4463,11 +4467,12 @@ void RGWPutObj::execute(optional_yield y)
     /* optional streaming checksum */
     try {
       cksum_filter =
-	rgw::putobj::RGWPutObj_Cksum::Factory(filter, *s->info.env);
+	rgw::putobj::RGWPutObj_Cksum::Factory(filter, *s->info.env, multipart_cksum_type);
     } catch (const rgw::io::Exception& e) {
       op_ret = -e.code().value();
       return;
     }
+
     if (cksum_filter) {
       filter = &*cksum_filter;
     }
@@ -4614,25 +4619,26 @@ void RGWPutObj::execute(optional_yield y)
 
   if (cksum_filter) {
     const auto& hdr = cksum_filter->header();
+    auto expected_ck = cksum_filter->expected(*s->info.env);
     auto cksum_verify =
       cksum_filter->verify(*s->info.env); // valid or no supplied cksum
     cksum = get<1>(cksum_verify);
-    if (std::get<0>(cksum_verify)) {
+    if ((!expected_ck) ||
+	std::get<0>(cksum_verify)) {
       buffer::list cksum_bl;
 
       ldpp_dout_fmt(this, 16,
 		    "{} checksum verified "
 		    "\n\tcomputed={} == \n\texpected={}",
 		    hdr.second,
 		    cksum->to_armor(),
-		    cksum_filter->expected(*s->info.env));
+		    (!!expected_ck) ? expected_ck : "(checksum unavailable)");
 
       cksum->encode(cksum_bl);
       emplace_attr(RGW_ATTR_CKSUM, std::move(cksum_bl));
     } else {
       /* content checksum mismatch */
       auto computed_ck = cksum->to_armor();
-      auto expected_ck = cksum_filter->expected(*s->info.env);
 
       ldpp_dout_fmt(this, 4,
 		    "{} content checksum mismatch"
@@ -4835,7 +4841,8 @@ void RGWPostObj::execute(optional_yield y)
     /* optional streaming checksum */
     try {
       cksum_filter =
-	rgw::putobj::RGWPutObj_Cksum::Factory(filter, *s->info.env);
+	rgw::putobj::RGWPutObj_Cksum::Factory(
+	  filter, *s->info.env, rgw::cksum::Type::none /* no override */);
     } catch (const rgw::io::Exception& e) {
       op_ret = -e.code().value();
       return;

src/rgw/rgw_op.h

@@ -1238,6 +1238,7 @@ class RGWPutObj : public RGWOp {
   std::string multipart_upload_id;
   std::string multipart_part_str;
   int multipart_part_num = 0;
+  rgw::cksum::Type multipart_cksum_type{rgw::cksum::Type::none};
   jspan_ptr multipart_trace;
 
   boost::optional<ceph::real_time> delete_at;