workbench: retry instance operations if they error out with 409: unable to queue the operation (#15158)

Author: bcreddy-gcp

mmv1/products/workbench/Instance.yaml

@@ -130,6 +130,8 @@ virtual_fields:
       Desired state of the Workbench Instance. Set this field to `ACTIVE` to start the Instance, and `STOPPED` to stop the Instance.
     type: String
     default_value: "ACTIVE"
+error_retry_predicates:
+  - 'transport_tpg.IsWorkbenchQueueError'
 parameters:
   - name: 'location'
     type: String

mmv1/third_party/terraform/transport/error_retry_predicates.go

@@ -304,6 +304,17 @@ func IsRepositoryGroupQueueError(err error) (bool, string) {
 	return false, ""
 }
 
+// Retry if Workbench operation returns a 409 with a specific message for
+// enqueued operations.
+func IsWorkbenchQueueError(err error) (bool, string) {
+	if gerr, ok := err.(*googleapi.Error); ok {
+		if gerr.Code == 409 && (strings.Contains(strings.ToLower(gerr.Body), "unable to queue the operation")) {
+			return true, "Waiting for other enqueued operations to finish"
+		}
+	}
+	return false, ""
+}
+
 // Retry if Monitoring operation returns a 409 with a specific message for
 // concurrent operations.
 func IsMonitoringConcurrentEditError(err error) (bool, string) {