Merge pull request ceph#62640 from ivancich/wip-admin-robust-rm-obj

rgw: add force option to `radosgw-admin object rm ...`

Reviewed-by: Casey Bodley <[email protected]>

Author: ivancich

doc/man/8/radosgw-admin.rst

@@ -135,22 +135,23 @@ as follows:
   Purge bucket index entries.
 
 :command:`object rm`
-  Remove an object.
+  Remove an S3/Swift object. Include "--yes-i-really-mean-it" to remove object's
+  entry from bucket index, for example if it's damaged.
 
 :command:`object stat`
-  Stat an object for its metadata.
+  Stat an S3/Swift object for its metadata.
 
 :command:`object manifest`
-  Display the manifest of RADOS objects containing the data.
+  Display the manifest of an S3/Swift object, producing a list of RADOS objects containing the data.
 
 :command:`object unlink`
-  Unlink object from bucket index.
+  Unlink S3/Swift object from bucket index.
 
 :command:`object rewrite`
-  Rewrite the specified object.
+  Rewrite the specified S3/Swift object.
 
 :command:`object reindex`
-  Add an object to its bucket's index. Used rarely for emergency repairs.
+  Add an S3/Swift object to its bucket's index. Used rarely for emergency repairs.
 
 :command:`objects expire`
   Run expired objects cleanup.

src/cls/rgw/cls_rgw.cc

@@ -2081,7 +2081,7 @@ static int rgw_bucket_unlink_instance(cls_method_context_t hctx, bufferlist *in,
              olh_key.name.c_str(), olh_key.instance.c_str(), olh_entry.delete_marker);
 
   if (olh_key == dest_key) {
-    /* this is the current head, need to update! */
+    /* this is the current head, need to update the OLH! */
     cls_rgw_obj_key next_key;
     bool found = false;
     ret = obj.find_next_key(&next_key, &found);

src/rgw/driver/rados/rgw_bucket.cc

@@ -145,12 +145,19 @@ bool rgw_bucket_object_check_filter(const std::string& oid)
   return rgw_obj_key::oid_to_key_in_ns(oid, &key, empty_ns);
 }
 
-int rgw_remove_object(const DoutPrefixProvider *dpp, rgw::sal::Driver* driver, rgw::sal::Bucket* bucket, rgw_obj_key& key, optional_yield y)
+int rgw_remove_object(const DoutPrefixProvider *dpp,
+		      rgw::sal::Driver* driver,
+		      rgw::sal::Bucket* bucket,
+		      rgw_obj_key& key,
+		      optional_yield y,
+		      const bool force)
 {
 
   std::unique_ptr<rgw::sal::Object> object = bucket->get_object(key);
 
-  return object->delete_object(dpp, y, rgw::sal::FLAG_LOG_OP, nullptr, nullptr);
+  const uint32_t possible_force_flag = force ? rgw::sal::FLAG_FORCE_OP : 0;
+
+  return object->delete_object(dpp, y, rgw::sal::FLAG_LOG_OP | possible_force_flag, nullptr, nullptr);
 }
 
 static void set_err_msg(std::string *sink, std::string msg)

src/rgw/driver/rados/rgw_bucket.h

@@ -199,7 +199,12 @@ auto create_archive_bucket_instance_metadata_handler(rgw::sal::Driver* driver,
     -> std::unique_ptr<RGWMetadataHandler>;
 
 
-extern int rgw_remove_object(const DoutPrefixProvider *dpp, rgw::sal::Driver* driver, rgw::sal::Bucket* bucket, rgw_obj_key& key, optional_yield y);
+extern int rgw_remove_object(const DoutPrefixProvider* dpp,
+			     rgw::sal::Driver* driver,
+			     rgw::sal::Bucket* bucket,
+			     rgw_obj_key& key,
+			     optional_yield y,
+			     const bool force = false);
 
 extern int rgw_object_get_attr(rgw::sal::Driver* driver, rgw::sal::Object* obj,
 			       const char* attr_name, bufferlist& out_bl,

src/rgw/driver/rados/rgw_rados.cc

@@ -6266,9 +6266,13 @@ struct tombstone_entry {
  * Delete an object.
  * bucket: name of the bucket storing the object
  * obj: name of the object to delete
+ * force: if b.i. entry exists but head object does not, still remove entry
  * Returns: 0 on success, -ERR# otherwise.
  */
-int RGWRados::Object::Delete::delete_obj(optional_yield y, const DoutPrefixProvider *dpp, bool log_op)
+int RGWRados::Object::Delete::delete_obj(optional_yield y,
+					 const DoutPrefixProvider* dpp,
+					 bool log_op,
+					 const bool force)
 {
   RGWRados *store = target->get_store();
   const rgw_obj& src_obj = target->get_obj();
@@ -6297,8 +6301,9 @@ int RGWRados::Object::Delete::delete_obj(optional_yield y, const DoutPrefixProvi
       }
 
       result.version_id = marker.key.instance;
-      if (result.version_id.empty())
+      if (result.version_id.empty()) {
         result.version_id = "null";
+      }
       result.delete_marker = true;
 
       struct rgw_bucket_dir_entry_meta meta;
@@ -6326,8 +6331,10 @@ int RGWRados::Object::Delete::delete_obj(optional_yield y, const DoutPrefixProvi
         return r;
       }
       result.delete_marker = dirent.is_delete_marker();
-      r = store->unlink_obj_instance(dpp, target->get_ctx(), target->get_bucket_info(), obj, params.olh_epoch,
-                                     y, params.bilog_flags, params.null_verid, params.zones_trace, add_log);
+      r = store->unlink_obj_instance(
+	dpp, target->get_ctx(), target->get_bucket_info(), obj,
+	params.olh_epoch, y, params.bilog_flags,
+	params.null_verid, params.zones_trace, add_log, force);
       if (r < 0) {
         return r;
       }
@@ -6349,7 +6356,7 @@ int RGWRados::Object::Delete::delete_obj(optional_yield y, const DoutPrefixProvi
     }
 
     return 0;
-  }
+  } // if versioned bucket
 
   rgw_rados_ref ref;
   int r = store->get_obj_head_ref(dpp, target->get_bucket_info(), obj, &ref);
@@ -6360,8 +6367,9 @@ int RGWRados::Object::Delete::delete_obj(optional_yield y, const DoutPrefixProvi
   RGWObjState *state;
   RGWObjManifest *manifest = nullptr;
   r = target->get_state(dpp, &state, &manifest, false, y);
-  if (r < 0)
+  if (r < 0) {
     return r;
+  }
 
   ObjectWriteOperation op;
 
@@ -6383,7 +6391,7 @@ int RGWRados::Object::Delete::delete_obj(optional_yield y, const DoutPrefixProvi
   }
   uint64_t obj_accounted_size = state->accounted_size;
 
-  if(params.abortmp) {
+  if (params.abortmp) {
     obj_accounted_size = params.parts_accounted_size;
   }
 
@@ -6409,13 +6417,20 @@ int RGWRados::Object::Delete::delete_obj(optional_yield y, const DoutPrefixProvi
   }
 
   if (!state->exists) {
-    target->invalidate_state();
-    return -ENOENT;
+    if (!force) {
+      target->invalidate_state();
+      return -ENOENT;
+    } else {
+      ldpp_dout(dpp, 5) << "WARNING: head for \"" << src_obj <<
+	"\" does not exist; will continue with deleting bucket "
+	"index entry(ies)" << dendl;
+    }
   }
 
   r = target->prepare_atomic_modification(dpp, op, false, NULL, NULL, NULL, true, false, y);
-  if (r < 0)
+  if (r < 0) {
     return r;
+  }
 
   RGWBucketInfo& bucket_info = target->get_bucket_info();
 
@@ -6429,10 +6444,10 @@ int RGWRados::Object::Delete::delete_obj(optional_yield y, const DoutPrefixProvi
     index_op.set_bilog_flags(params.bilog_flags | RGW_BILOG_NULL_VERSION);
   }
 
-
   r = index_op.prepare(dpp, CLS_RGW_OP_DEL, &state->write_tag, y);
-  if (r < 0)
+  if (r < 0) {
     return r;
+  }
 
   store->remove_rgw_head_obj(op);
 
@@ -6477,8 +6492,9 @@ int RGWRados::Object::Delete::delete_obj(optional_yield y, const DoutPrefixProvi
     target->invalidate_state();
   }
 
-  if (r < 0)
+  if (r < 0) {
     return r;
+  }
 
   /* update quota cache */
   store->quota_handler->update_stats(params.bucket_owner, obj.bucket, -1, 0, obj_accounted_size);
@@ -6495,7 +6511,8 @@ int RGWRados::delete_obj(const DoutPrefixProvider *dpp,
                          uint16_t bilog_flags,
                          const real_time& expiration_time,
                          rgw_zone_set *zones_trace,
-                         bool log_op)
+                         bool log_op,
+                         const bool force) // force removal even if head object is broken
 {
   RGWRados::Object del_target(this, bucket_info, obj_ctx, obj);
   RGWRados::Object::Delete del_op(&del_target);
@@ -6507,7 +6524,7 @@ int RGWRados::delete_obj(const DoutPrefixProvider *dpp,
   del_op.params.zones_trace = zones_trace;
   del_op.params.null_verid = null_verid;
 
-  return del_op.delete_obj(y, dpp, log_op ? rgw::sal::FLAG_LOG_OP : 0);
+  return del_op.delete_obj(y, dpp, log_op, force);
 }
 
 int RGWRados::delete_raw_obj(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, optional_yield y)
@@ -8618,7 +8635,7 @@ int RGWRados::bucket_index_unlink_instance(const DoutPrefixProvider *dpp,
   }
 
   return 0;
-}
+} // bucket_index_unlink_instance
 
 int RGWRados::bucket_index_read_olh_log(const DoutPrefixProvider *dpp,
                                         RGWBucketInfo& bucket_info, RGWObjState& state,
@@ -8813,9 +8830,10 @@ int RGWRados::apply_olh_log(const DoutPrefixProvider *dpp,
 			    std::map<uint64_t, std::vector<rgw_bucket_olh_log_entry> >& log,
 			    uint64_t *plast_ver,
 			    optional_yield y,
-          bool null_verid,
-          rgw_zone_set* zones_trace,
-          bool log_op)
+			    bool null_verid,
+			    rgw_zone_set* zones_trace,
+			    bool log_op,
+			    const bool force)
 {
   if (log.empty()) {
     return 0;
@@ -8928,7 +8946,9 @@ int RGWRados::apply_olh_log(const DoutPrefixProvider *dpp,
        liter != remove_instances.end(); ++liter) {
     cls_rgw_obj_key& key = *liter;
     rgw_obj obj_instance(bucket, key);
-    int ret = delete_obj(dpp, obj_ctx, bucket_info, obj_instance, 0, y, null_verid, RGW_BILOG_FLAG_VERSIONED_OP, ceph::real_time(), zones_trace, log_op);
+    int ret = delete_obj(dpp, obj_ctx, bucket_info, obj_instance, 0, y,
+			 null_verid, RGW_BILOG_FLAG_VERSIONED_OP,
+			 ceph::real_time(), zones_trace, log_op, force);
     if (ret < 0 && ret != -ENOENT) {
       ldpp_dout(dpp, 0) << "ERROR: delete_obj() returned " << ret << " obj_instance=" << obj_instance << dendl;
       return ret;
@@ -8942,7 +8962,6 @@ int RGWRados::apply_olh_log(const DoutPrefixProvider *dpp,
     return r;
   }
 
-
   if (need_to_remove) {
     string olh_tag(state.olh_tag.c_str(), state.olh_tag.length());
     r = clear_olh(dpp, obj_ctx, obj, bucket_info, ref, olh_tag, last_ver, y);
@@ -9032,7 +9051,15 @@ int RGWRados::clear_olh(const DoutPrefixProvider *dpp,
 /*
  * read olh log and apply it
  */
-int RGWRados::update_olh(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, RGWObjState *state, RGWBucketInfo& bucket_info, const rgw_obj& obj, optional_yield y, rgw_zone_set *zones_trace, bool null_verid, bool log_op)
+int RGWRados::update_olh(const DoutPrefixProvider* dpp,
+			 RGWObjectCtx& obj_ctx,
+			 RGWObjState* state,
+			 RGWBucketInfo& bucket_info,
+			 const rgw_obj& obj, optional_yield y,
+			 rgw_zone_set* zones_trace,
+			 bool null_verid,
+			 bool log_op,
+			 const bool force)
 {
   map<uint64_t, vector<rgw_bucket_olh_log_entry> > log;
   bool is_truncated;
@@ -9043,7 +9070,9 @@ int RGWRados::update_olh(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, R
     if (ret < 0) {
       return ret;
     }
-    ret = apply_olh_log(dpp, obj_ctx, *state, bucket_info, obj, state->olh_tag, log, &ver_marker, y, null_verid, zones_trace, log_op);
+    ret = apply_olh_log(dpp, obj_ctx, *state, bucket_info, obj,
+			state->olh_tag, log, &ver_marker, y,
+			null_verid, zones_trace, log_op, force);
     if (ret < 0) {
       return ret;
     }
@@ -9144,8 +9173,17 @@ int RGWRados::set_olh(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx,
   return 0;
 }
 
-int RGWRados::unlink_obj_instance(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, RGWBucketInfo& bucket_info, const rgw_obj& target_obj,
-                                  uint64_t olh_epoch, optional_yield y, uint16_t bilog_flags, bool null_verid, rgw_zone_set *zones_trace, bool log_op)
+int RGWRados::unlink_obj_instance(const DoutPrefixProvider* dpp,
+				  RGWObjectCtx& obj_ctx,
+				  RGWBucketInfo& bucket_info,
+				  const rgw_obj& target_obj,
+                                  uint64_t olh_epoch,
+				  optional_yield y,
+				  uint16_t bilog_flags,
+				  bool null_verid,
+				  rgw_zone_set* zones_trace,
+				  bool log_op,
+				  const bool force)
 {
   string op_tag;
 
@@ -9190,31 +9228,36 @@ int RGWRados::unlink_obj_instance(const DoutPrefixProvider *dpp, RGWObjectCtx& o
       bilog_flags = bilog_flags | RGW_BILOG_FLAG_VERSIONED_OP;
     }
 
-    ret = bucket_index_unlink_instance(dpp, bucket_info, target_obj, op_tag, olh_tag, olh_epoch, y, bilog_flags, zones_trace, log_op);
+    ret = bucket_index_unlink_instance(dpp, bucket_info, target_obj,
+				       op_tag, olh_tag, olh_epoch, y,
+				       bilog_flags, zones_trace, log_op);
     if (ret < 0) {
       olh_cancel_modification(dpp, bucket_info, *state, olh_obj, op_tag, y);
-      ldpp_dout(dpp, 20) << "bucket_index_unlink_instance() target_obj=" << target_obj << " returned " << ret << dendl;
+      ldpp_dout(dpp, 20) << "bucket_index_unlink_instance() target_obj=" <<
+	target_obj << " returned " << ret << dendl;
       if (ret == -ECANCELED) {
         continue;
       }
       // it's possible that the pending xattr from this op prevented the olh
       // object from being cleaned by another thread that was deleting the last
       // existing version. We invoke a best-effort update_olh here to handle this case.
-      int r = update_olh(dpp, obj_ctx, state, bucket_info, olh_obj, y, zones_trace, null_verid, log_op);
+      int r = update_olh(dpp, obj_ctx, state, bucket_info, olh_obj, y,
+			 zones_trace, null_verid, log_op, force);
       if (r < 0 && r != -ECANCELED) {
         ldpp_dout(dpp, 20) << "update_olh() target_obj=" << olh_obj << " returned " << r << dendl;
       }
       return ret;
-    }
+    } // if error in bucket_index_unlink_instance call
     break;
-  }
+  } // cancel retry loop
 
   if (i == MAX_ECANCELED_RETRY) {
     ldpp_dout(dpp, 0) << "ERROR: exceeded max ECANCELED retries, aborting (EIO)" << dendl;
     return -EIO;
   }
 
-  ret = update_olh(dpp, obj_ctx, state, bucket_info, olh_obj, y, zones_trace, null_verid, log_op);
+  ret = update_olh(dpp, obj_ctx, state, bucket_info, olh_obj, y,
+		   zones_trace, null_verid, log_op, force);
   if (ret == -ECANCELED) { /* already did what we needed, no need to retry, raced with another user */
     return 0;
   }
@@ -11322,7 +11365,7 @@ int RGWRados::delete_obj_aio(const DoutPrefixProvider *dpp, const rgw_obj& obj,
     }
   }
   return ret;
-}
+} // delete_obj_aio
 
 void objexp_hint_entry::generate_test_instances(list<objexp_hint_entry*>& o)
 {