fix(obs): method added (#622)

fix(obs): cleanup

Reviewed-by: Anton Sidelnikov

Author: RusselSand

otcextensions/sdk/obs/v1/_proxy.py

@@ -23,6 +23,8 @@
 from otcextensions.sdk import sdk_proxy
 from otcextensions.sdk.obs.v1 import container as _container
 from otcextensions.sdk.obs.v1 import obj as _obj
+from openstack import utils as os_utils
+
 
 DEFAULT_OBJECT_SEGMENT_SIZE = 1073741824  # 1GB
 DEFAULT_MAX_FILE_SIZE = int((5 * 1024 * 1024 * 1024 + 2) / 2)
@@ -772,3 +774,126 @@ def _abort_multipart_upload(
         """
         url = f'{endpoint}?uploadId={upload_id}'
         return self.delete(url, requests_auth=self._get_req_auth(endpoint))
+
+    def wait_for_delete_object(self, obj, container=None,
+                               interval=2, wait=300):
+        """Waits for an object to be deleted.
+
+        :param obj: The value can be the name of an object or a
+                    :class:`~otcextensions.sdk.obs.v1.obj.Object` instance.
+        :param container: The value can be the name of a container or a
+               :class:`~otcextensions.sdk.obs.v1.container.Container`.
+        :param int interval: The time in seconds to wait between checks.
+        :param int wait: The maximum time in seconds to wait for the object
+                         to be deleted.
+
+        :returns: The last head response from the server.
+        :raises: :class:`~openstack.exceptions.ResourceTimeout` if the
+                 object is not deleted in the specified time.
+        """
+        obj = self._get_resource(_obj.Object, obj)
+        container = self._get_container_name(obj, container)
+        for _ in os_utils.iterate_timeout(
+                timeout=wait,
+                message=f"Timeout waiting for {obj.name} "
+                        f"in {container} to delete",
+                wait=interval,
+        ):
+            try:
+                obj.fetch(self, container=container, skip_cache=True)
+            except exceptions.NotFoundException:
+                return obj
+
+        raise exceptions.ResourceTimeout(
+            f"Object {obj.name} in {container} was not "
+            f"deleted in {wait} seconds"
+        )
+
+    def wait_for_delete_container(self, container, interval=2, wait=180):
+        """Waits for a container to be deleted.
+
+        :param container: The value can be either the name of a container or a
+                      :class:`~otcextensions.sdk.obs.v1.container.Container`
+                      instance.
+        :param int interval: The time in seconds to wait between checks.
+        :param int wait: The maximum time in seconds to wait for the container
+                         to be deleted.
+        :returns: The last head response from the server.
+        :raises: :class:`~openstack.exceptions.ResourceTimeout` if the
+                 container is not deleted in the specified time.
+        """
+        container = self._get_resource(_container.Container, container)
+
+        for _ in os_utils.iterate_timeout(
+                timeout=wait,
+                message=f"Timeout waiting for container"
+                        f" {container.name} to delete",
+                wait=interval,
+        ):
+            try:
+                self.get_container(container.name)
+            except exceptions.NotFoundException:
+                return container
+        raise exceptions.ResourceTimeout(
+            f"Container {container.name} was not deleted in {wait} seconds"
+        )
+
+    def _get_cleanup_dependencies(self):
+        return {
+            'obs': {
+                'before': ['network']
+            }
+        }
+
+    def _service_cleanup(
+            self,
+            dry_run=True,
+            client_status_queue=None,
+            identified_resources=None,
+            filters=None,
+            resource_evaluation_fn=None,
+            skip_resources=None,
+            cont_name=None,
+    ):
+        pass
+        containers = []
+        for container in self.containers():
+            objects = []
+            for obj in self.objects(container=container.name):
+                need_delete = self._service_cleanup_del_res(
+                    lambda o: self.delete_object(o, container=container.name),
+                    obj,
+                    dry_run=dry_run,
+                    client_status_queue=client_status_queue,
+                    identified_resources=identified_resources,
+                    filters=filters,
+                    resource_evaluation_fn=resource_evaluation_fn
+                )
+                if not dry_run and need_delete:
+                    objects.append(obj)
+            for obj in objects:
+                try:
+                    self.wait_for_delete_object(obj, container=container.name)
+                except Exception as e:
+                    self.log.warning(
+                        f"Failed to delete object {obj.name} "
+                        f"from {container.name}: {e}"
+                    )
+            need_delete = self._service_cleanup_del_res(
+                self.delete_container,
+                container,
+                dry_run=dry_run,
+                client_status_queue=client_status_queue,
+                identified_resources=identified_resources,
+                filters=filters,
+                resource_evaluation_fn=resource_evaluation_fn
+            )
+            if not dry_run and need_delete:
+                containers.append(container)
+        for container in containers:
+            try:
+                self.wait_for_delete_container(container)
+            except Exception as e:
+                self.log.warning(
+                    f"Failed to delete container {container.name}: {e}"
+                )

otcextensions/sdk/obs/v1/container.py

@@ -29,6 +29,7 @@ class Container(_base.BaseResource):
     resource_key = 'Bucket'
 
     allow_get = True
+    allow_fetch = True
     allow_head = True
     allow_list = True
     allow_create = True

otcextensions/sdk/obs/v1/obj.py

@@ -32,6 +32,7 @@ class Object(_base.BaseResource):
 
     allow_create = True
     allow_get = True
+    allow_fetch = True
     allow_commit = True
     allow_delete = True
     allow_list = True

otcextensions/tests/functional/sdk/obs/v1/test_cleanup.py

@@ -0,0 +1,52 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+from otcextensions.tests.functional import base
+import uuid
+
+
+class TestObsCleanup(base.BaseFunctionalTest):
+    uuid_v4 = uuid.uuid4().hex[:8]
+    bucket_name = 'obs-test-' + uuid_v4
+    object_name = f'obs{uuid_v4}.object'
+    folder_name = f'folder{uuid_v4}/'
+    data = str(uuid.uuid4())
+
+    def setUp(self):
+        super(TestObsCleanup, self).setUp()
+        self.client = self.conn.obs
+        self.container = self.client.create_container(
+            name=self.bucket_name,
+            storage_acl='public-read-write',
+            storage_class='STANDARD'
+        )
+        self.object = self.client.create_object(
+            container=self.container,
+            name=self.object_name,
+            data=self.data
+        )
+        self.folder = self.client.create_object(
+            container=self.container,
+            name=self.folder_name
+        )
+        self.nested_object_name = f"{self.folder_name}nest_{self.uuid_v4}.txt"
+        self.nested_object = self.client.create_object(
+            container=self.container,
+            name=self.nested_object_name,
+            data="nested test data"
+        )
+
+    def test_cleanup(self):
+        containers = list(self.client.containers())
+        self.assertGreaterEqual(len(containers), 1)
+        self.client._service_cleanup(dry_run=False, cont_name=self.bucket_name)
+        containers = list(self.client.containers())
+        self.assertEqual(len(containers), 0)

releasenotes/notes/obs-cleanup-e1fc241e44836c4b.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    OBS cleanup implemented