Merge pull request ceph#55259 from linuxbox2/wip-object-attributes

rgw: implement GetObjectAttributes

Author: mattbenjamin

src/rgw/driver/posix/rgw_sal_posix.cc

@@ -2893,6 +2893,14 @@ int POSIXObject::copy_object(const ACLOwner& owner,
   return dobj->set_obj_attrs(dpp, &attrs, nullptr, y, rgw::sal::FLAG_LOG_OP);
 }
 
+int POSIXObject::list_parts(const DoutPrefixProvider* dpp, CephContext* cct,
+			    int max_parts, int marker, int* next_marker,
+			    bool* truncated, list_parts_each_t each_func,
+			    optional_yield y)
+{
+  return -EOPNOTSUPP;
+}
+
 int POSIXObject::load_obj_state(const DoutPrefixProvider* dpp, optional_yield y, bool follow_olh)
 {
   int ret = stat(dpp);

src/rgw/driver/posix/rgw_sal_posix.h

@@ -653,6 +653,13 @@ class POSIXObject : public StoreObject {
                const DoutPrefixProvider* dpp, optional_yield y) override;
   virtual RGWAccessControlPolicy& get_acl(void) override { return acls; }
   virtual int set_acl(const RGWAccessControlPolicy& acl) override { acls = acl; return 0; }
+
+  /** If multipart, enumerate (a range [marker..marker+[min(max_parts, parts_count-1)] of) parts of the object */
+  virtual int list_parts(const DoutPrefixProvider* dpp, CephContext* cct,
+			 int max_parts, int marker, int* next_marker,
+			 bool* truncated, list_parts_each_t each_func,
+			 optional_yield y) override;
+
   virtual int load_obj_state(const DoutPrefixProvider* dpp, optional_yield y, bool follow_olh = true) override;
   virtual int set_obj_attrs(const DoutPrefixProvider* dpp, Attrs* setattrs,
 			    Attrs* delattrs, optional_yield y, uint32_t flags) override;

src/rgw/driver/rados/rgw_rados.cc

@@ -6962,13 +6962,13 @@ int RGWRados::set_attrs(const DoutPrefixProvider *dpp, RGWObjectCtx* octx, RGWBu
   }
 
   return 0;
-}
+} /* RGWRados::set_attrs() */
 
-static int get_part_obj_state(const DoutPrefixProvider* dpp, optional_yield y,
-                              RGWRados* store, RGWBucketInfo& bucket_info,
-                              RGWObjectCtx* rctx, RGWObjManifest* manifest,
-                              int part_num, int* parts_count, bool prefetch,
-                              RGWObjState** pstate, RGWObjManifest** pmanifest)
+int RGWRados::get_part_obj_state(const DoutPrefixProvider* dpp, optional_yield y,
+				 RGWRados* store, RGWBucketInfo& bucket_info,
+				 RGWObjectCtx* rctx, RGWObjManifest* manifest,
+				 int part_num, int* parts_count, bool prefetch,
+				 RGWObjState** pstate, RGWObjManifest** pmanifest)
 {
   if (!manifest) {
     return -ERR_INVALID_PART;
@@ -7047,6 +7047,9 @@ static int get_part_obj_state(const DoutPrefixProvider* dpp, optional_yield y,
 
   // update the object size
   sm->state.size = part_manifest.get_obj_size();
+  if (!sm->state.attrset.count(RGW_ATTR_COMPRESSION)) {
+    sm->state.accounted_size = sm->state.size;
+  }
 
   *pmanifest = &part_manifest;
   return 0;

src/rgw/driver/rados/rgw_rados.h

@@ -1071,6 +1071,12 @@ class RGWRados
     }; // class RGWRados::Bucket::List
   }; // class RGWRados::Bucket
 
+  static int get_part_obj_state(const DoutPrefixProvider* dpp, optional_yield y,
+		       RGWRados* store, RGWBucketInfo& bucket_info,
+		       RGWObjectCtx* rctx, RGWObjManifest* manifest,
+		       int part_num, int* parts_count, bool prefetch,
+		       RGWObjState** pstate, RGWObjManifest** pmanifest);
+
   int on_last_entry_in_listing(const DoutPrefixProvider *dpp,
                                RGWBucketInfo& bucket_info,
                                const std::string& obj_prefix,

src/rgw/driver/rados/rgw_sal_rados.cc

@@ -2471,7 +2471,108 @@ bool RadosObject::is_sync_completed(const DoutPrefixProvider* dpp,
 
   const rgw_bi_log_entry& earliest_marker = entries.front();
   return earliest_marker.timestamp > obj_mtime;
-}
+} /* is_sync_completed */
+
+int RadosObject::list_parts(const DoutPrefixProvider* dpp, CephContext* cct,
+			   int max_parts, int marker, int* next_marker,
+			   bool* truncated, list_parts_each_t each_func,
+			   optional_yield y)
+{
+  int ret{0};
+
+  /* require an object with a manifest, so call to get_obj_state() must precede this */
+  if (! manifest) {
+    return -EINVAL;
+  }
+
+  RGWObjManifest::obj_iterator end = manifest->obj_end(dpp);
+  if (end.get_cur_part_id() == 0) { // not multipart
+    ldpp_dout(dpp, 20) << __func__ << " object does not have a multipart manifest"
+		       << dendl;
+    return 0;
+  }
+
+  auto end_part_id = end.get_cur_part_id();
+  auto parts_count = (end_part_id == 1) ? 1 : end_part_id - 1;
+  if (marker > (parts_count - 1)) {
+    return 0;
+  }
+
+  RGWObjManifest::obj_iterator part_iter = manifest->obj_begin(dpp);
+
+  if (marker != 0) {
+    ldpp_dout_fmt(dpp, 20,
+		  "{} seeking to part #{} in the object manifest",
+		  __func__, marker);
+
+    part_iter  = manifest->obj_find_part(dpp, marker + 1);
+
+    if (part_iter == end) {
+      ldpp_dout_fmt(dpp, 5,
+		    "{} failed to find part #{} in the object manifest",
+		    __func__, marker + 1);
+      return 0;
+    }
+  }
+
+  RGWObjectCtx& obj_ctx = get_ctx();
+  RGWBucketInfo& bucket_info = get_bucket()->get_info();
+
+  Object::Part obj_part{};
+  for (; part_iter != manifest->obj_end(dpp); ++part_iter) {
+
+    /* we're only interested in the first object in each logical part */
+    auto cur_part_id = part_iter.get_cur_part_id();
+    if (cur_part_id == obj_part.part_number) {
+      continue;
+    }
+
+    if (max_parts < 1) {
+      *truncated = true;
+      break;
+    }
+
+    /* get_part_obj_state alters the passed manifest** to point to a part
+     * manifest, which we don't want to leak out here */
+    RGWObjManifest* obj_m = manifest;
+    RGWObjState* astate;
+    bool part_prefetch = false;
+    ret = RGWRados::get_part_obj_state(dpp, y, store->getRados(), bucket_info, &obj_ctx,
+				       obj_m, cur_part_id, &parts_count,
+				       part_prefetch, &astate, &obj_m);
+
+    if (ret < 0) {
+      ldpp_dout_fmt(dpp, 4,
+		    "{} get_part_obj_state() failed ret={}",
+		    __func__, ret);
+      break;
+    }
+
+    obj_part.part_number = part_iter.get_cur_part_id();
+    obj_part.part_size = astate->accounted_size;
+
+    if (auto iter = astate->attrset.find(RGW_ATTR_CKSUM);
+	iter != astate->attrset.end()) {
+          try {
+	    rgw::cksum::Cksum part_cksum;
+	    auto ck_iter = iter->second.cbegin();
+	    part_cksum.decode(ck_iter);
+	    obj_part.cksum = std::move(part_cksum);
+	  } catch (buffer::error& err) {
+	    ldpp_dout_fmt(dpp, 4,
+			  "WARN: {} could not decode stored cksum, "
+			  "caught buffer::error",
+			  __func__);
+	  }
+    }
+
+    each_func(obj_part);
+    *next_marker = ++marker;
+    --max_parts;
+  } /* each part */
+  
+  return ret;
+} /* RadosObject::list_parts */
 
 int RadosObject::load_obj_state(const DoutPrefixProvider* dpp, optional_yield y, bool follow_olh)
 {

src/rgw/driver/rados/rgw_sal_rados.h

@@ -592,12 +592,18 @@ class RadosObject : public StoreObject {
       StoreObject::set_compressed();
     }
 
-
     virtual bool is_sync_completed(const DoutPrefixProvider* dpp,
       const ceph::real_time& obj_mtime) override;
     /* For rgw_admin.cc */
     RGWObjState& get_state() { return state; }
     virtual int load_obj_state(const DoutPrefixProvider* dpp, optional_yield y, bool follow_olh = true) override;
+
+    /** If multipart, enumerate (a range [marker..marker+[min(max_parts, parts_count-1)] of) parts of the object */
+    virtual int list_parts(const DoutPrefixProvider* dpp, CephContext* cct,
+			   int max_parts, int marker, int* next_marker,
+			   bool* truncated, list_parts_each_t each_func,
+			   optional_yield y) override;
+
     virtual int set_obj_attrs(const DoutPrefixProvider* dpp, Attrs* setattrs, Attrs* delattrs, optional_yield y, uint32_t flags) override;
     virtual int get_obj_attrs(optional_yield y, const DoutPrefixProvider* dpp, rgw_obj* target_obj = NULL) override;
     virtual int modify_obj_attrs(const char* attr_name, bufferlist& attr_val, optional_yield y, const DoutPrefixProvider* dpp) override;

src/rgw/rgw_cksum_pipe.cc

@@ -18,6 +18,7 @@
 #include <string>
 #include <fmt/format.h>
 #include <boost/algorithm/string.hpp>
+#include "rgw_cksum.h"
 #include "rgw_common.h"
 #include "common/dout.h"
 #include "rgw_client_io.h"
@@ -34,7 +35,8 @@ namespace rgw::putobj {
   {}
 
   std::unique_ptr<RGWPutObj_Cksum> RGWPutObj_Cksum::Factory(
-    rgw::sal::DataProcessor* next, const RGWEnv& env)
+    rgw::sal::DataProcessor* next, const RGWEnv& env,
+    rgw::cksum::Type override_type)
   {
     /* look for matching headers */
     auto algo_header = cksum_algorithm_hdr(env);
@@ -49,6 +51,13 @@ namespace rgw::putobj {
       throw rgw::io::Exception(EINVAL, std::system_category());
     }
     /* no checksum header */
+    if (override_type != rgw::cksum::Type::none) {
+      /* XXXX safe? do we need to fixup env as well? */
+      auto algo_header = cksum_algorithm_hdr(override_type);
+      return
+	std::make_unique<RGWPutObj_Cksum>(
+			   next, override_type, std::move(algo_header));
+    }
     return std::unique_ptr<RGWPutObj_Cksum>();
   }
 

src/rgw/rgw_cksum_pipe.h

@@ -20,6 +20,7 @@
 #include <tuple>
 #include <cstring>
 #include <boost/algorithm/string/case_conv.hpp>
+#include "rgw_cksum.h"
 #include "rgw_cksum_digest.h"
 #include "rgw_common.h"
 #include "rgw_putobj.h"
@@ -29,6 +30,38 @@ namespace rgw::putobj {
   namespace cksum = rgw::cksum;
   using cksum_hdr_t = std::pair<const char*, const char*>;
 
+  static inline const cksum_hdr_t cksum_algorithm_hdr(rgw::cksum::Type t) {
+    static constexpr std::string_view hdr =
+      "HTTP_X_AMZ_SDK_CHECKSUM_ALGORITHM";
+    using rgw::cksum::Type;
+    switch (t) {
+    case Type::sha256:
+      return cksum_hdr_t(hdr.data(), "SHA256");
+      break;
+    case Type::crc32:
+      return cksum_hdr_t(hdr.data(), "CRC32");
+      break;
+    case Type::crc32c:
+      return cksum_hdr_t(hdr.data(), "CRC32C");
+      break;
+    case Type::xxh3:
+      return cksum_hdr_t(hdr.data(), "XX3");
+      break;
+    case Type::sha1:
+      return cksum_hdr_t(hdr.data(), "SHA1");
+      break;
+    case Type::sha512:
+      return cksum_hdr_t(hdr.data(), "SHA512");
+      break;
+    case Type::blake3:
+      return cksum_hdr_t(hdr.data(), "BLAKE3");
+      break;
+    default:
+      break;
+    };
+    return cksum_hdr_t(nullptr, nullptr);;
+  }
+
   static inline const cksum_hdr_t cksum_algorithm_hdr(const RGWEnv& env) {
     /* If the individual checksum value you provide through
        x-amz-checksum-algorithm doesn't match the checksum algorithm
@@ -102,7 +135,8 @@ namespace rgw::putobj {
     using VerifyResult = std::tuple<bool, const cksum::Cksum&>;
 
     static std::unique_ptr<RGWPutObj_Cksum> Factory(
-      rgw::sal::DataProcessor* next, const RGWEnv&);
+      rgw::sal::DataProcessor* next, const RGWEnv&,
+      rgw::cksum::Type override_type);
 
     RGWPutObj_Cksum(rgw::sal::DataProcessor* next, rgw::cksum::Type _type,
 		    cksum_hdr_t&& _hdr);

src/rgw/rgw_iam_policy.cc

@@ -94,6 +94,8 @@ static const actpair actpairs[] =
  { "s3:GetPublicAccessBlock", s3GetPublicAccessBlock },
  { "s3:GetObjectAcl", s3GetObjectAcl },
  { "s3:GetObject", s3GetObject },
+ { "s3:GetObjectAttributes", s3GetObjectAttributes },
+ { "s3:GetObjectVersionAttributes", s3GetObjectVersionAttributes },
  { "s3:GetObjectTorrent", s3GetObjectTorrent },
  { "s3:GetObjectVersionAcl", s3GetObjectVersionAcl },
  { "s3:GetObjectVersion", s3GetObjectVersion },
@@ -1335,6 +1337,7 @@ const char* action_bit_string(uint64_t action) {
 
   case s3ListBucketVersions:
     return "s3:ListBucketVersions";
+
   case s3ListAllMyBuckets:
     return "s3:ListAllMyBuckets";
 
@@ -1479,6 +1482,12 @@ const char* action_bit_string(uint64_t action) {
   case s3BypassGovernanceRetention:
     return "s3:BypassGovernanceRetention";
 
+  case s3GetObjectAttributes:
+    return "s3:GetObjectAttributes";
+
+  case s3GetObjectVersionAttributes:
+    return "s3:GetObjectVersionAttributes";
+
   case s3DescribeJob:
     return "s3:DescribeJob";
 

src/rgw/rgw_iam_policy.h

@@ -115,6 +115,8 @@ enum {
   s3GetBucketEncryption,
   s3PutBucketEncryption,
   s3DescribeJob,
+  s3GetObjectAttributes,
+  s3GetObjectVersionAttributes,
   s3All,
 
   s3objectlambdaGetObject,
@@ -247,6 +249,8 @@ inline int op_to_perm(std::uint64_t op) {
   case s3GetObjectVersionTagging:
   case s3GetObjectRetention:
   case s3GetObjectLegalHold:
+  case s3GetObjectAttributes:
+  case s3GetObjectVersionAttributes:
   case s3ListAllMyBuckets:
   case s3ListBucket:
   case s3ListBucketMultipartUploads: