Merge pull request ceph#63043 from cbodley/wip-71115-tentacle

tentacle: rgw/sts: correcting authentication in case s3 ops are directed to a primary from secondary after assumerole.

Reviewed-by: Seena Fallah <[email protected]>

Author: cbodley

PendingReleaseNotes

@@ -138,6 +138,13 @@
   allowed.  `rbd trash mv` command now behaves the same way as `rbd rm` in this
   scenario.
 
+* RGW: Replication policies now validate permissions using `s3:ReplicateObject`,
+  `s3:ReplicateDelete`, and `s3:ReplicateTags` for destination buckets. For source
+  buckets, both `s3:GetObjectVersionForReplication` and `s3:GetObject(Version)`
+  are supported. Actions like `s3:GetObjectAcl`, `s3:GetObjectLegalHold`, and
+  `s3:GetObjectRetention` are also considered when fetching the source object.
+  Replication of tags is controlled by the `s3:GetObject(Version)Tagging` permission.
+
 >=19.2.1
 
 * CephFS: Command `fs subvolume create` now allows tagging subvolumes through option

qa/suites/rgw/multisite/overrides.yaml

@@ -20,5 +20,7 @@ overrides:
         rgw sync obj etag verify: true
         rgw sync meta inject err probability: 0
         rgw sync data inject err probability: 0
+        rgw s3 auth use sts: true
+        rgw sts key: abcdefghijklmnoq
   rgw:
     compression type: random

src/rgw/driver/rados/rgw_bucket_sync.h

@@ -415,3 +415,31 @@ class RGWBucketSyncPolicyHandler {
   }
 };
 
+struct rgw_bucket_sync_pair_info {
+  RGWBucketSyncFlowManager::pipe_handler handler; /* responsible for sync filters */
+  rgw_bucket_shard source_bs;
+  rgw_bucket dest_bucket;
+};
+
+inline std::ostream& operator<<(std::ostream& out, const rgw_bucket_sync_pair_info& p) {
+  if (p.source_bs.bucket == p.dest_bucket) {
+    return out << p.source_bs;
+  }
+  return out << p.source_bs << "->" << p.dest_bucket;
+}
+
+struct rgw_bucket_sync_pipe {
+  rgw_bucket_sync_pair_info info;
+  RGWBucketInfo source_bucket_info;
+  std::map<std::string, bufferlist> source_bucket_attrs;
+  RGWBucketInfo dest_bucket_info;
+  std::map<std::string, bufferlist> dest_bucket_attrs;
+
+  RGWBucketSyncFlowManager::pipe_rules_ref& get_rules() {
+    return info.handler.rules;
+  }
+};
+
+inline std::ostream& operator<<(std::ostream& out, const rgw_bucket_sync_pipe& p) {
+  return out << p.info;
+}

src/rgw/driver/rados/rgw_cr_rados.cc

@@ -12,6 +12,7 @@
 #include "rgw_cr_rest.h"
 #include "rgw_rest_conn.h"
 #include "rgw_rados.h"
+#include "rgw_data_sync.h"
 
 #include "services/svc_zone.h"
 #include "services/svc_zone_utils.h"
@@ -812,7 +813,8 @@ int RGWAsyncFetchRemoteObj::_send_request(const DoutPrefixProvider *dpp)
   std::optional<uint64_t> bytes_transferred;
   const req_context rctx{dpp, null_yield, nullptr};
   int r = store->getRados()->fetch_remote_obj(obj_ctx,
-                       user_id.value_or(rgw_user()),
+                       NULL, /* uid */
+                       user_id ? &*user_id : nullptr, /* replication uid */
                        NULL, /* req_info */
                        source_zone,
                        dest_obj.get_obj(),
@@ -843,7 +845,8 @@ int RGWAsyncFetchRemoteObj::_send_request(const DoutPrefixProvider *dpp)
                        stat_dest_obj,
                        source_trace_entry,
                        &zones_trace,
-                       &bytes_transferred);
+                       &bytes_transferred,
+                       keep_tags);
 
   if (r < 0) {
     ldpp_dout(dpp, 0) << "store->fetch_remote_obj() returned r=" << r << dendl;
@@ -873,7 +876,6 @@ int RGWAsyncStatRemoteObj::_send_request(const DoutPrefixProvider *dpp)
 {
   RGWObjectCtx obj_ctx(store);
 
-  string user_id;
   char buf[16];
   snprintf(buf, sizeof(buf), ".%lld", (long long)store->getRados()->instance_id());
 
@@ -882,7 +884,7 @@ int RGWAsyncStatRemoteObj::_send_request(const DoutPrefixProvider *dpp)
 
   int r = store->getRados()->stat_remote_obj(dpp,
                        obj_ctx,
-                       rgw_user(user_id),
+                       nullptr, /* user_id */
                        nullptr, /* req_info */
                        source_zone,
                        src_obj,
@@ -925,9 +927,55 @@ int RGWAsyncRemoveObj::_send_request(const DoutPrefixProvider *dpp)
     return 0;
   }
 
-  RGWAccessControlPolicy policy;
+  RGWObjTags obj_tags;
+  bufferlist bl_tag;
+  if (obj->get_attr(RGW_ATTR_TAGS, bl_tag)) {
+    auto bliter = bl_tag.cbegin();
+    try {
+      obj_tags.decode(bliter);
+    } catch (buffer::error &err) {
+      ldpp_dout(dpp, 0) << "ERROR: " << __func__ << ": caught buffer::error couldn't decode TagSet " << dendl;
+      return -EIO;
+    }
+  }
+
+  rgw_sync_pipe_params params;
+  if (!sync_pipe.info.handler.find_obj_params(obj->get_key(),
+                                              obj_tags.get_tags(),
+                                              &params)) {
+    return -ERR_PRECONDITION_FAILED;
+  }
+
+  if (params.mode == rgw_sync_pipe_params::MODE_USER) {
+    std::optional<RGWUserPermHandler> user_perms;
+    RGWUserPermHandler::Bucket dest_bucket_perms;
+
+    if (!params.user.has_value()) {
+      ldpp_dout(dpp, 20) << "ERROR: " << __func__ << ": user level sync but user param not set" << dendl;
+      return -EPERM;
+    }
+    user_perms.emplace(dpp, store, dpp->get_cct(), *params.user);
+
+    ret = user_perms->init();
+    if (ret < 0) {
+      ldpp_dout(dpp, 0) << "ERROR: " << __func__ << ": failed to init user perms for uid=" << *params.user << " ret=" << ret << dendl;
+      return ret;
+    }
+
+    ret = user_perms->init_bucket(sync_pipe.dest_bucket_info, sync_pipe.dest_bucket_attrs, &dest_bucket_perms);
+    if (ret < 0) {
+      ldpp_dout(dpp, 0) << "ERROR: " << __func__ << ": failed to init bucket perms for uid=" << *params.user << " bucket=" << bucket->get_key() << " ret=" << ret << dendl;
+      return ret;
+    }
+
+    if (!dest_bucket_perms.verify_bucket_permission(obj->get_key(), rgw::IAM::s3ReplicateDelete)) {
+      ldpp_dout(dpp, 20) << "ERROR: " << __func__ << ": user does not have permission to delete object" << dendl;
+      return -EPERM;
+    }
+  }
 
   /* decode policy */
+  RGWAccessControlPolicy policy;
   bufferlist bl;
   if (obj->get_attr(RGW_ATTR_ACL, bl)) {
     auto bliter = bl.cbegin();

src/rgw/driver/rados/rgw_cr_rados.h

@@ -8,6 +8,7 @@
 #include "rgw_coroutine.h"
 #include "rgw_sal.h"
 #include "rgw_sal_rados.h"
+#include "rgw_bucket_sync.h"
 #include "common/WorkQueue.h"
 #include "common/Throttle.h"
 
@@ -1129,6 +1130,7 @@ class RGWAsyncFetchRemoteObj : public RGWAsyncRadosRequest {
   rgw_zone_set zones_trace;
   PerfCounters* counters;
   const DoutPrefixProvider *dpp;
+  bool keep_tags;
 
 protected:
   int _send_request(const DoutPrefixProvider *dpp) override;
@@ -1148,7 +1150,8 @@ class RGWAsyncFetchRemoteObj : public RGWAsyncRadosRequest {
                          const rgw_zone_set_entry& source_trace_entry,
                          rgw_zone_set *_zones_trace,
                          PerfCounters* counters,
-                         const DoutPrefixProvider *dpp)
+                         const DoutPrefixProvider *dpp,
+                         bool _keep_tags)
     : RGWAsyncRadosRequest(caller, cn), store(_store),
       source_zone(_source_zone),
       user_id(_user_id),
@@ -1163,7 +1166,8 @@ class RGWAsyncFetchRemoteObj : public RGWAsyncRadosRequest {
       stat_follow_olh(_stat_follow_olh),
       source_trace_entry(source_trace_entry),
       counters(counters),
-      dpp(dpp)
+      dpp(dpp),
+      keep_tags(_keep_tags)
   {
     if (_zones_trace) {
       zones_trace = *_zones_trace;
@@ -1199,6 +1203,7 @@ class RGWFetchRemoteObjCR : public RGWSimpleCoroutine {
   rgw_zone_set *zones_trace;
   PerfCounters* counters;
   const DoutPrefixProvider *dpp;
+  bool keep_tags;
 
 public:
   RGWFetchRemoteObjCR(RGWAsyncRadosProcessor *_async_rados, rgw::sal::RadosStore* _store,
@@ -1216,7 +1221,8 @@ class RGWFetchRemoteObjCR : public RGWSimpleCoroutine {
                       const rgw_zone_set_entry& source_trace_entry,
                       rgw_zone_set *_zones_trace,
                       PerfCounters* counters,
-                      const DoutPrefixProvider *dpp)
+                      const DoutPrefixProvider *dpp,
+                      bool _keep_tags)
     : RGWSimpleCoroutine(_store->ctx()), cct(_store->ctx()),
       async_rados(_async_rados), store(_store),
       source_zone(_source_zone),
@@ -1232,7 +1238,7 @@ class RGWFetchRemoteObjCR : public RGWSimpleCoroutine {
       req(NULL),
       stat_follow_olh(_stat_follow_olh),
       source_trace_entry(source_trace_entry),
-      zones_trace(_zones_trace), counters(counters), dpp(dpp) {}
+      zones_trace(_zones_trace), counters(counters), dpp(dpp), keep_tags(_keep_tags) {}
 
 
   ~RGWFetchRemoteObjCR() override {
@@ -1250,7 +1256,7 @@ class RGWFetchRemoteObjCR : public RGWSimpleCoroutine {
     req = new RGWAsyncFetchRemoteObj(this, stack->create_completion_notifier(), store,
     source_zone, user_id, src_bucket, dest_placement_rule, dest_bucket_info,
                                      key, dest_key, versioned_epoch, copy_if_newer, filter,
-                                     stat_follow_olh, source_trace_entry, zones_trace, counters, dpp);
+                                     stat_follow_olh, source_trace_entry, zones_trace, counters, dpp, keep_tags);
     async_rados->queue(req);
     return 0;
   }
@@ -1360,7 +1366,9 @@ class RGWStatRemoteObjCR : public RGWSimpleCoroutine {
 class RGWAsyncRemoveObj : public RGWAsyncRadosRequest {
   const DoutPrefixProvider *dpp;
   rgw::sal::RadosStore* store;
+  CephContext *cct;
   rgw_zone_id source_zone;
+  rgw_bucket_sync_pipe& sync_pipe;
 
   std::unique_ptr<rgw::sal::Bucket> bucket;
   std::unique_ptr<rgw::sal::Object> obj;
@@ -1379,33 +1387,35 @@ class RGWAsyncRemoveObj : public RGWAsyncRadosRequest {
   int _send_request(const DoutPrefixProvider *dpp) override;
 public:
   RGWAsyncRemoveObj(const DoutPrefixProvider *_dpp, RGWCoroutine *caller, RGWAioCompletionNotifier *cn, 
-                         rgw::sal::RadosStore* _store,
-                         const rgw_zone_id& _source_zone,
-                         RGWBucketInfo& _bucket_info,
-                         const rgw_obj_key& _key,
-                         const std::string& _owner,
-                         const std::string& _owner_display_name,
-                         bool _versioned,
-                         uint64_t _versioned_epoch,
-                         bool _delete_marker,
-                         bool _if_older,
-                         real_time& _timestamp,
-                         rgw_zone_set* _zones_trace) : RGWAsyncRadosRequest(caller, cn), dpp(_dpp), store(_store),
-                                                      source_zone(_source_zone),
-                                                      owner(_owner),
-                                                      owner_display_name(_owner_display_name),
-                                                      versioned(_versioned),
-                                                      versioned_epoch(_versioned_epoch),
-                                                      del_if_older(_if_older),
-                                                      timestamp(_timestamp) {
+                    rgw::sal::RadosStore* _store,
+                    CephContext *_cct,
+                    const rgw_zone_id& _source_zone,
+                    rgw_bucket_sync_pipe& _sync_pipe,
+                    const rgw_obj_key& _key,
+                    const std::string& _owner,
+                    const std::string& _owner_display_name,
+                    bool _versioned,
+                    uint64_t _versioned_epoch,
+                    bool _delete_marker,
+                    bool _if_older,
+                    real_time& _timestamp,
+                    rgw_zone_set* _zones_trace) : RGWAsyncRadosRequest(caller, cn), dpp(_dpp), store(_store), cct(_cct),
+                                                  source_zone(_source_zone),
+                                                  sync_pipe(_sync_pipe),
+                                                  owner(_owner),
+                                                  owner_display_name(_owner_display_name),
+                                                  versioned(_versioned),
+                                                  versioned_epoch(_versioned_epoch),
+                                                  del_if_older(_if_older),
+                                                  timestamp(_timestamp) {
     if (_delete_marker) {
       marker_version_id = _key.instance;
     }
 
     if (_zones_trace) {
       zones_trace = *_zones_trace;
     }
-    bucket = store->get_bucket(_bucket_info);
+    bucket = store->get_bucket(sync_pipe.dest_bucket_info);
     obj = bucket->get_object(_key);
   }
 };
@@ -1417,7 +1427,7 @@ class RGWRemoveObjCR : public RGWSimpleCoroutine {
   rgw::sal::RadosStore* store;
   rgw_zone_id source_zone;
 
-  RGWBucketInfo bucket_info;
+  rgw_bucket_sync_pipe& sync_pipe;
 
   rgw_obj_key key;
   bool versioned;
@@ -1436,7 +1446,7 @@ class RGWRemoveObjCR : public RGWSimpleCoroutine {
 public:
   RGWRemoveObjCR(const DoutPrefixProvider *_dpp, RGWAsyncRadosProcessor *_async_rados, rgw::sal::RadosStore* _store,
                       const rgw_zone_id& _source_zone,
-                      RGWBucketInfo& _bucket_info,
+                      rgw_bucket_sync_pipe& _sync_pipe,
                       const rgw_obj_key& _key,
                       bool _versioned,
                       uint64_t _versioned_epoch,
@@ -1447,7 +1457,7 @@ class RGWRemoveObjCR : public RGWSimpleCoroutine {
                       rgw_zone_set *_zones_trace) : RGWSimpleCoroutine(_store->ctx()), dpp(_dpp), cct(_store->ctx()),
                                        async_rados(_async_rados), store(_store),
                                        source_zone(_source_zone),
-                                       bucket_info(_bucket_info),
+                                       sync_pipe(_sync_pipe),
                                        key(_key),
                                        versioned(_versioned),
                                        versioned_epoch(_versioned_epoch),
@@ -1477,7 +1487,7 @@ class RGWRemoveObjCR : public RGWSimpleCoroutine {
   }
 
   int send_request(const DoutPrefixProvider *dpp) override {
-    req = new RGWAsyncRemoveObj(dpp, this, stack->create_completion_notifier(), store, source_zone, bucket_info,
+    req = new RGWAsyncRemoveObj(dpp, this, stack->create_completion_notifier(), store, cct, source_zone, sync_pipe,
                                 key, owner, owner_display_name, versioned, versioned_epoch,
                                 delete_marker, del_if_older, timestamp, zones_trace);
     async_rados->queue(req);