Merge pull request ceph#60916 from shashalu/fix-data-corruption-ETIMEDOUT

rgw: fix data corruption when rados op return ETIMEDOUT

Reviewed-by: Casey Bodley <[email protected]>

Author: ivancich

src/rgw/driver/rados/rgw_putobj_processor.cc

@@ -597,6 +597,11 @@ int MultipartObjectProcessor::complete(
   }
 
   if (r < 0) {
+    if (r == -ETIMEDOUT) {
+      // The meta_obj_ref write may eventually succeed, clear the set of objects for deletion. if it
+      // doesn't ever succeed, we'll orphan any tail objects as if we'd crashed before that write
+      writer.clear_written();
+    }
     return r == -ENOENT ? -ERR_NO_SUCH_UPLOAD : r;
   }
 
@@ -783,6 +788,11 @@ int AppendObjectProcessor::complete(
 			attrs, rctx, writer.get_trace(),
 			flags & rgw::sal::FLAG_LOG_OP);
   if (r < 0) {
+      if (r == -ETIMEDOUT) {
+      // The head object write may eventually succeed, clear the set of objects for deletion. if it
+      // doesn't ever succeed, we'll orphan any tail objects as if we'd crashed before that write
+      writer.clear_written();
+    }
     return r;
   }
   if (!obj_op.meta.canceled) {

src/rgw/driver/rados/rgw_rados.cc

@@ -3390,12 +3390,17 @@ int RGWRados::Object::Write::_do_write_meta(uint64_t size, uint64_t accounted_si
   return 0;
 
 done_cancel:
-  int ret = index_op->cancel(rctx.dpp, meta.remove_objs, rctx.y, log_op);
-  if (ret < 0) {
-    ldpp_dout(rctx.dpp, 0) << "ERROR: index_op.cancel() returned ret=" << ret << dendl;
-  }
+  // if r == -ETIMEDOUT, rgw can't determine whether or not the rados op succeeded
+  // we shouldn't be calling index_op->cancel() in this case
+  // Instead, we should leave that pending entry in the index so than bucket listing can recover with check_disk_state() and cls_rgw_suggest_changes()
+  if (r != -ETIMEDOUT) {
+    int ret = index_op->cancel(rctx.dpp, meta.remove_objs, rctx.y, log_op);
+    if (ret < 0) {
+      ldpp_dout(rctx.dpp, 0) << "ERROR: index_op.cancel() returned ret=" << ret << dendl;
+    }
 
-  meta.canceled = true;
+    meta.canceled = true;
+  }
 
   /* we lost in a race. There are a few options:
    * - existing object was rewritten (ECANCELED)