openstack destroy: account for BULK DELETE limits on object-storage

Some object-storage instances may be set to have a limit to the LIST
operation that is higher to the limit to the BULK DELETE operation. On
those clouds, objects in the BULK DELETE call beyond the limit are
silently ignored. As a consequence, the call to destroy the container
fails and object deletion is re-queued after a growing waiting time,
potentially dilating deletion by hours.

With this change, object bulk deletion is put in a loop. After checking
that no errors were encountered, we reduce the BULK DELETE list by the
number of processed objects, and send it back to the server. As a
consequence, the object deletion routines should only complete when the
container is emtpy, thus avoiding the 409 error that causes a retry.

Author: pierreprinetti

pkg/destroy/openstack/openstack.go

@@ -1099,21 +1099,31 @@ func deleteContainers(opts *clientconfig.ClientOpts, filter Filter, logger logru
 						return false, err
 					}
 					queue.Add(func() {
-						logger.Debugf("Initiating bulk deletion of %d objects in container %q", len(objectsOnPage), container)
-						resp, err := objects.BulkDelete(conn, container, objectsOnPage).Extract()
-						if err != nil {
-							errCh <- err
-							return
-						}
-						if len(resp.Errors) > 0 {
-							// Convert resp.Errors to golang errors.
-							// Each error is represented by a list of 2 strings, where the first one
-							// is the object name, and the second one contains an error message.
-							for _, objectError := range resp.Errors {
-								errCh <- fmt.Errorf("cannot delete object %q: %s", objectError[0], objectError[1])
+						for len(objectsOnPage) > 0 {
+							logger.Debugf("Initiating bulk deletion of %d objects in container %q", len(objectsOnPage), container)
+							resp, err := objects.BulkDelete(conn, container, objectsOnPage).Extract()
+							if err != nil {
+								errCh <- err
+								return
+							}
+							if len(resp.Errors) > 0 {
+								// Convert resp.Errors to golang errors.
+								// Each error is represented by a list of 2 strings, where the first one
+								// is the object name, and the second one contains an error message.
+								for _, objectError := range resp.Errors {
+									errCh <- fmt.Errorf("cannot delete object %q: %s", objectError[0], objectError[1])
+								}
+								logger.Debugf("Terminating object deletion routine with error. Deleted %d objects out of %d.", resp.NumberDeleted, len(objectsOnPage))
 							}
+
+							// Some object-storage instances may be set to have a limit to the LIST operation
+							// that is higher to the limit to the BULK DELETE operation. On those clouds, objects
+							// in the BULK DELETE call beyond the limit are silently ignored. In this loop, after
+							// checking that no errors were encountered, we reduce the BULK DELETE list by the
+							// number of processed objects, and send it back to the server if it's not empty.
+							objectsOnPage = objectsOnPage[resp.NumberDeleted+resp.NumberNotFound:]
 						}
-						logger.Debugf("Terminating object deletion routine. Deleted %d objects out of %d.", resp.NumberDeleted, len(objectsOnPage))
+						logger.Debugf("Terminating object deletion routine.")
 					})
 					return true, nil
 				})