feat(storage): add MoveObject functionality to JSON and gRPC (#14936)

* feat(storage): add MoveObject functionality to JSON and gRPC

* add moveobject integration test and update testbench version

* checkers

* add patchbucket call to integration test

* use folder enabled bucket

* create folder bucket in emulator

* make non-pure virtual to fix abi issue

Author: ddelgrosso1

ci/cloudbuild/builds/lib/integration.sh

@@ -31,7 +31,7 @@ source module ci/lib/io.sh
 export PATH="${HOME}/.local/bin:${PATH}"
 python3 -m pip uninstall -y --quiet googleapis-storage-testbench
 python3 -m pip install --upgrade --user --quiet --disable-pip-version-check \
-  "git+https://github.com/googleapis/storage-testbench@v0.46.0"
+  "git+https://github.com/googleapis/storage-testbench@v0.52.0"
 
 # Some of the tests will need a valid roots.pem file.
 rm -f /dev/shm/roots.pem

ci/lib/run_gcs_httpbin_emulator_utils.sh

@@ -127,4 +127,8 @@ create_testbench_resources() {
     curl -s -o /dev/null -X POST --data-binary @- \
       -H "Content-Type: application/json" \
       "${CLOUD_STORAGE_EMULATOR_ENDPOINT}/storage/v1/b?project=${GOOGLE_CLOUD_PROJECT}"
+  printf '{"name": "%s"}' "${GOOGLE_CLOUD_CPP_STORAGE_TEST_FOLDER_BUCKET_NAME}" |
+    curl -s -o /dev/null -X POST --data-binary @- \
+      -H "Content-Type: application/json" \
+      "${CLOUD_STORAGE_EMULATOR_ENDPOINT}/storage/v1/b?project=${GOOGLE_CLOUD_PROJECT}"
 }

google/cloud/storage/client.h

@@ -1372,6 +1372,39 @@ class Client {
     return connection_->UpdateObject(request);
   }
 
+  /**
+   * Moves an existing object to a new or existing object within a HNS enabled
+   * bucket.
+   *
+   * @param bucket_name the name of the bucket in which to move the object. The
+   * bucket must be HNS enabled.
+   * @param source_object_name the name of the source object to move.
+   * @param destination_object_name the destination name of the object after the
+   * move is completed.
+   * @param options a list of optional query parameters and/or request headers.
+   *     Valid types for this operation include
+   *      `IfSourceGenerationMatch`, `IfSourceGenerationNotMatch`,
+   * `IfSourceMetagenerationMatch`, `IfSourceMetagenerationNotMatch`,
+   * `IfGenerationMatch`, `IfGenerationNotMatch`, `IfMetagenerationMatch`
+   *      `IfMetagenerationNotMatch`, `projection`.
+   *
+   * @par Idempotency
+   * This operation is only idempotent if restricted by pre-conditions.
+   */
+  template <typename... Options>
+  StatusOr<ObjectMetadata> MoveObject(std::string bucket_name,
+                                      std::string source_object_name,
+                                      std::string destination_object_name,
+                                      Options&&... options) {
+    google::cloud::internal::OptionsSpan const span(
+        SpanOptions(std::forward<Options>(options)...));
+    internal::MoveObjectRequest request(std::move(bucket_name),
+                                        std::move(source_object_name),
+                                        std::move(destination_object_name));
+    request.set_multiple_options(std::forward<Options>(options)...);
+    return connection_->MoveObject(request);
+  }
+
   /**
    * Patches the metadata in a Google Cloud Storage Object.
    *

google/cloud/storage/idempotency_policy.cc

@@ -92,6 +92,10 @@ bool AlwaysRetryIdempotencyPolicy::IsIdempotent(
     internal::UpdateObjectRequest const&) const {
   return true;
 }
+bool AlwaysRetryIdempotencyPolicy::IsIdempotent(
+    internal::MoveObjectRequest const&) const {
+  return true;
+}
 bool AlwaysRetryIdempotencyPolicy::IsIdempotent(
     internal::PatchObjectRequest const&) const {
   return true;
@@ -341,6 +345,11 @@ bool StrictIdempotencyPolicy::IsIdempotent(
          request.HasOption<IfMetagenerationMatch>();
 }
 
+bool StrictIdempotencyPolicy::IsIdempotent(
+    internal::MoveObjectRequest const& request) const {
+  return request.HasOption<IfGenerationMatch>();
+}
+
 bool StrictIdempotencyPolicy::IsIdempotent(
     internal::PatchObjectRequest const& request) const {
   return request.HasOption<IfMatchEtag>() ||

google/cloud/storage/idempotency_policy.h

@@ -102,6 +102,9 @@ class IdempotencyPolicy {
       internal::DeleteObjectRequest const& request) const = 0;
   virtual bool IsIdempotent(
       internal::UpdateObjectRequest const& request) const = 0;
+  virtual bool IsIdempotent(internal::MoveObjectRequest const&) const {
+    return false;
+  };
   virtual bool IsIdempotent(
       internal::PatchObjectRequest const& request) const = 0;
   virtual bool IsIdempotent(
@@ -238,6 +241,7 @@ class AlwaysRetryIdempotencyPolicy : public IdempotencyPolicy {
       internal::DeleteObjectRequest const& request) const override;
   bool IsIdempotent(
       internal::UpdateObjectRequest const& request) const override;
+  bool IsIdempotent(internal::MoveObjectRequest const& request) const override;
   bool IsIdempotent(internal::PatchObjectRequest const& request) const override;
   bool IsIdempotent(
       internal::ComposeObjectRequest const& request) const override;
@@ -370,6 +374,7 @@ class StrictIdempotencyPolicy : public IdempotencyPolicy {
       internal::DeleteObjectRequest const& request) const override;
   bool IsIdempotent(
       internal::UpdateObjectRequest const& request) const override;
+  bool IsIdempotent(internal::MoveObjectRequest const& request) const override;
   bool IsIdempotent(internal::PatchObjectRequest const& request) const override;
   bool IsIdempotent(
       internal::ComposeObjectRequest const& request) const override;

google/cloud/storage/idempotency_policy_test.cc

@@ -231,6 +231,21 @@ TEST(StrictIdempotencyPolicyTest, UpdateObjectIfMetagenerationMatch) {
   EXPECT_TRUE(policy.IsIdempotent(request));
 }
 
+TEST(StrictIdempotencyPolicyTest, MoveObject) {
+  StrictIdempotencyPolicy policy;
+  internal::MoveObjectRequest request(
+      "test-bucket-name", "test-src-object-name", "test-dst-object-name");
+  EXPECT_FALSE(policy.IsIdempotent(request));
+}
+
+TEST(StrictIdempotencyPolicyTest, MoveObjectIfGenerationMatch) {
+  StrictIdempotencyPolicy policy;
+  internal::MoveObjectRequest request(
+      "test-bucket-name", "test-src-object-name", "test-dst-object-name");
+  request.set_option(IfGenerationMatch(7));
+  EXPECT_TRUE(policy.IsIdempotent(request));
+}
+
 TEST(StrictIdempotencyPolicyTest, PatchObject) {
   StrictIdempotencyPolicy policy;
   internal::PatchObjectRequest request("test-bucket-name", "test-object-name",

google/cloud/storage/internal/connection_impl.cc

@@ -451,6 +451,22 @@ StatusOr<ObjectMetadata> StorageConnectionImpl::UpdateObject(
       google::cloud::internal::CurrentOptions(), request, __func__);
 }
 
+StatusOr<ObjectMetadata> StorageConnectionImpl::MoveObject(
+    MoveObjectRequest const& request) {
+  auto const idempotency = current_idempotency_policy().IsIdempotent(request)
+                               ? Idempotency::kIdempotent
+                               : Idempotency::kNonIdempotent;
+  return RestRetryLoop(
+      current_retry_policy(), current_backoff_policy(), idempotency,
+      [token = MakeIdempotencyToken(), this](
+          rest_internal::RestContext& context, Options const& options,
+          auto const& request) {
+        context.AddHeader(kIdempotencyTokenHeader, token);
+        return stub_->MoveObject(context, options, request);
+      },
+      google::cloud::internal::CurrentOptions(), request, __func__);
+}
+
 StatusOr<ObjectMetadata> StorageConnectionImpl::PatchObject(
     PatchObjectRequest const& request) {
   auto const idempotency = current_idempotency_policy().IsIdempotent(request)

google/cloud/storage/internal/connection_impl.h

@@ -80,6 +80,8 @@ class StorageConnectionImpl
   StatusOr<EmptyResponse> DeleteObject(DeleteObjectRequest const&) override;
   StatusOr<ObjectMetadata> UpdateObject(
       UpdateObjectRequest const& request) override;
+  StatusOr<ObjectMetadata> MoveObject(
+      MoveObjectRequest const& request) override;
   StatusOr<ObjectMetadata> PatchObject(
       PatchObjectRequest const& request) override;
   StatusOr<ObjectMetadata> ComposeObject(

google/cloud/storage/internal/connection_impl_object_copy_test.cc

@@ -122,6 +122,34 @@ TEST(StorageConnectionImpl, RewriteObjectPermanentFailure) {
   EXPECT_THAT(permanent.captured_authority_options(), RetryLoopUsesOptions());
 }
 
+TEST(StorageConnectionImpl, MoveObjectTooManyFailures) {
+  auto transient = MockRetryClientFunction(TransientError());
+  auto mock = std::make_unique<MockGenericStub>();
+  EXPECT_CALL(*mock, options);
+  EXPECT_CALL(*mock, MoveObject).Times(3).WillRepeatedly(transient);
+  auto client =
+      StorageConnectionImpl::Create(std::move(mock), RetryTestOptions());
+  google::cloud::internal::OptionsSpan span(client->options());
+  auto response = client->MoveObject(MoveObjectRequest()).status();
+  EXPECT_THAT(response, StoppedOnTooManyTransients("MoveObject"));
+  EXPECT_THAT(transient.captured_tokens(), RetryLoopUsesSingleToken());
+  EXPECT_THAT(transient.captured_authority_options(), RetryLoopUsesOptions());
+}
+
+TEST(StorageConnectionImpl, MoveObjectPermanentFailure) {
+  auto permanent = MockRetryClientFunction(PermanentError());
+  auto mock = std::make_unique<MockGenericStub>();
+  EXPECT_CALL(*mock, options);
+  EXPECT_CALL(*mock, MoveObject).WillOnce(permanent);
+  auto client =
+      StorageConnectionImpl::Create(std::move(mock), RetryTestOptions());
+  google::cloud::internal::OptionsSpan span(client->options());
+  auto response = client->MoveObject(MoveObjectRequest()).status();
+  EXPECT_THAT(response, StoppedOnPermanentError("MoveObject"));
+  EXPECT_THAT(permanent.captured_tokens(), RetryLoopUsesSingleToken());
+  EXPECT_THAT(permanent.captured_authority_options(), RetryLoopUsesOptions());
+}
+
 }  // namespace
 }  // namespace internal
 GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END

google/cloud/storage/internal/generic_stub.h

@@ -121,6 +121,9 @@ class GenericStub {
   virtual StatusOr<storage::ObjectMetadata> UpdateObject(
       rest_internal::RestContext&, Options const&,
       storage::internal::UpdateObjectRequest const&) = 0;
+  virtual StatusOr<storage::ObjectMetadata> MoveObject(
+      rest_internal::RestContext&, Options const&,
+      storage::internal::MoveObjectRequest const&) = 0;
   virtual StatusOr<storage::ObjectMetadata> PatchObject(
       rest_internal::RestContext&, Options const&,
       storage::internal::PatchObjectRequest const&) = 0;