os_ops::rmdirs (local, remote) was refactored (#207)

* os_ops::rmdirs (local, remote) was refactored

LocalOperations::rmdirs
 - parameter 'retries' was remaned with 'attempts'
 - if ignore_errors we raise an error

RemoteOperations::rmdirs
 - parameter 'verbose' was removed
 - method returns bool
 - we prevent to delete a file

* [TestRemoteOperations] New tests for rmdirs are added.

* test_pg_ctl_wait_option (local, remote) is corrected

Author: dmitry-lipetsk

testgres/operations/local_ops.py

@@ -174,7 +174,8 @@ def makedirs(self, path, remove_existing=False):
         except FileExistsError:
             pass
 
-    def rmdirs(self, path, ignore_errors=True, retries=3, delay=1):
+    # [2025-02-03] Old name of parameter attempts is "retries".
+    def rmdirs(self, path, ignore_errors=True, attempts=3, delay=1):
         """
         Removes a directory and its contents, retrying on failure.
 
@@ -183,18 +184,38 @@ def rmdirs(self, path, ignore_errors=True, retries=3, delay=1):
         :param retries: Number of attempts to remove the directory.
         :param delay: Delay between attempts in seconds.
         """
-        for attempt in range(retries):
+        assert type(path) == str  # noqa: E721
+        assert type(ignore_errors) == bool  # noqa: E721
+        assert type(attempts) == int  # noqa: E721
+        assert type(delay) == int or type(delay) == float  # noqa: E721
+        assert attempts > 0
+        assert delay >= 0
+
+        attempt = 0
+        while True:
+            assert attempt < attempts
+            attempt += 1
             try:
-                rmtree(path, ignore_errors=ignore_errors)
-                if not os.path.exists(path):
-                    return True
+                rmtree(path)
             except FileNotFoundError:
-                return True
+                pass
             except Exception as e:
-                logging.error(f"Error: Failed to remove directory {path} on attempt {attempt + 1}: {e}")
-            time.sleep(delay)
-        logging.error(f"Error: Failed to remove directory {path} after {retries} attempts.")
-        return False
+                if attempt < attempt:
+                    errMsg = "Failed to remove directory {0} on attempt {1} ({2}): {3}".format(
+                        path, attempt, type(e).__name__, e
+                    )
+                    logging.warning(errMsg)
+                    time.sleep(delay)
+                    continue
+
+                assert attempt == attempts
+                if not ignore_errors:
+                    raise
+
+                return False
+
+            # OK!
+            return True
 
     def listdir(self, path):
         return os.listdir(path)

testgres/operations/remote_ops.py

@@ -4,6 +4,7 @@
 import subprocess
 import tempfile
 import io
+import logging
 
 # we support both pg8000 and psycopg2
 try:
@@ -222,20 +223,45 @@ def makedirs(self, path, remove_existing=False):
             raise Exception("Couldn't create dir {} because of error {}".format(path, error))
         return result
 
-    def rmdirs(self, path, verbose=False, ignore_errors=True):
+    def rmdirs(self, path, ignore_errors=True):
         """
         Remove a directory in the remote server.
         Args:
         - path (str): The path to the directory to be removed.
-        - verbose (bool): If True, return exit status, result, and error.
         - ignore_errors (bool): If True, do not raise error if directory does not exist.
         """
-        cmd = "rm -rf {}".format(path)
-        exit_status, result, error = self.exec_command(cmd, verbose=True)
-        if verbose:
-            return exit_status, result, error
-        else:
-            return result
+        assert type(path) == str  # noqa: E721
+        assert type(ignore_errors) == bool  # noqa: E721
+
+        # ENOENT = 2 - No such file or directory
+        # ENOTDIR = 20 - Not a directory
+
+        cmd1 = [
+            "if", "[", "-d", path, "]", ";",
+            "then", "rm", "-rf", path, ";",
+            "elif", "[", "-e", path, "]", ";",
+            "then", "{", "echo", "cannot remove '" + path + "': it is not a directory", ">&2", ";", "exit", "20", ";", "}", ";",
+            "else", "{", "echo", "directory '" + path + "' does not exist", ">&2", ";", "exit", "2", ";", "}", ";",
+            "fi"
+        ]
+
+        cmd2 = ["sh", "-c", subprocess.list2cmdline(cmd1)]
+
+        try:
+            self.exec_command(cmd2, encoding=Helpers.GetDefaultEncoding())
+        except ExecUtilException as e:
+            if e.exit_code == 2:  # No such file or directory
+                return True
+
+            if not ignore_errors:
+                raise
+
+            errMsg = "Failed to remove directory {0} ({1}): {2}".format(
+                path, type(e).__name__, e
+            )
+            logging.warning(errMsg)
+            return False
+        return True
 
     def listdir(self, path):
         """

tests/test_remote.py

@@ -99,9 +99,9 @@ def test_makedirs_and_rmdirs_success(self):
         assert not os.path.exists(path)
         assert not self.operations.path_exists(path)
 
-    def test_makedirs_and_rmdirs_failure(self):
+    def test_makedirs_failure(self):
         """
-        Test makedirs and rmdirs for directory creation and removal failure.
+        Test makedirs for failure.
         """
         # Try to create a directory in a read-only location
         path = "/root/test_dir"
@@ -110,16 +110,84 @@ def test_makedirs_and_rmdirs_failure(self):
         with pytest.raises(Exception):
             self.operations.makedirs(path)
 
-        # Test rmdirs
-        while True:
-            try:
-                self.operations.rmdirs(path, verbose=True)
-            except ExecUtilException as e:
-                assert e.message == "Utility exited with non-zero code (1). Error: `rm: cannot remove '/root/test_dir': Permission denied`"
-                assert type(e.error) == bytes  # noqa: E721
-                assert e.error.strip() == b"rm: cannot remove '/root/test_dir': Permission denied"
-                break
-            raise Exception("We wait an exception!")
+    def test_rmdirs(self):
+        path = self.operations.mkdtemp()
+        assert os.path.exists(path)
+
+        assert self.operations.rmdirs(path, ignore_errors=False) is True
+        assert not os.path.exists(path)
+
+    def test_rmdirs__01_with_subfolder(self):
+        # folder with subfolder
+        path = self.operations.mkdtemp()
+        assert os.path.exists(path)
+
+        dir1 = os.path.join(path, "dir1")
+        assert not os.path.exists(dir1)
+
+        self.operations.makedirs(dir1)
+        assert os.path.exists(dir1)
+
+        assert self.operations.rmdirs(path, ignore_errors=False) is True
+        assert not os.path.exists(path)
+        assert not os.path.exists(dir1)
+
+    def test_rmdirs__02_with_file(self):
+        # folder with file
+        path = self.operations.mkdtemp()
+        assert os.path.exists(path)
+
+        file1 = os.path.join(path, "file1.txt")
+        assert not os.path.exists(file1)
+
+        self.operations.touch(file1)
+        assert os.path.exists(file1)
+
+        assert self.operations.rmdirs(path, ignore_errors=False) is True
+        assert not os.path.exists(path)
+        assert not os.path.exists(file1)
+
+    def test_rmdirs__03_with_subfolder_and_file(self):
+        # folder with subfolder and file
+        path = self.operations.mkdtemp()
+        assert os.path.exists(path)
+
+        dir1 = os.path.join(path, "dir1")
+        assert not os.path.exists(dir1)
+
+        self.operations.makedirs(dir1)
+        assert os.path.exists(dir1)
+
+        file1 = os.path.join(dir1, "file1.txt")
+        assert not os.path.exists(file1)
+
+        self.operations.touch(file1)
+        assert os.path.exists(file1)
+
+        assert self.operations.rmdirs(path, ignore_errors=False) is True
+        assert not os.path.exists(path)
+        assert not os.path.exists(dir1)
+        assert not os.path.exists(file1)
+
+    def test_rmdirs__try_to_delete_nonexist_path(self):
+        path = "/root/test_dir"
+
+        assert self.operations.rmdirs(path, ignore_errors=False) is True
+
+    def test_rmdirs__try_to_delete_file(self):
+        path = self.operations.mkstemp()
+        assert os.path.exists(path)
+
+        with pytest.raises(ExecUtilException) as x:
+            self.operations.rmdirs(path, ignore_errors=False)
+
+        assert os.path.exists(path)
+        assert type(x.value) == ExecUtilException   # noqa: E721
+        assert x.value.message == "Utility exited with non-zero code (20). Error: `cannot remove '" + path + "': it is not a directory`"
+        assert type(x.value.error) == str  # noqa: E721
+        assert x.value.error.strip() == "cannot remove '" + path + "': it is not a directory"
+        assert type(x.value.exit_code) == int  # noqa: E721
+        assert x.value.exit_code == 20
 
     def test_listdir(self):
         """

tests/test_simple.py

@@ -428,16 +428,61 @@ def test_backup_wrong_xlog_method(self):
                 node.backup(xlog_method='wrong')
 
     def test_pg_ctl_wait_option(self):
-        with get_new_node() as node:
-            node.init().start(wait=False)
-            while True:
-                try:
-                    node.stop(wait=False)
-                    break
-                except ExecUtilException:
-                    # it's ok to get this exception here since node
-                    # could be not started yet
-                    pass
+        C_MAX_ATTEMPTS = 50
+
+        node = get_new_node()
+        assert node.status() == testgres.NodeStatus.Uninitialized
+        node.init()
+        assert node.status() == testgres.NodeStatus.Stopped
+        node.start(wait=False)
+        nAttempt = 0
+        while True:
+            if nAttempt == C_MAX_ATTEMPTS:
+                raise Exception("Could not stop node.")
+
+            nAttempt += 1
+
+            if nAttempt > 1:
+                logging.info("Wait 1 second.")
+                time.sleep(1)
+                logging.info("")
+
+            logging.info("Try to stop node. Attempt #{0}.".format(nAttempt))
+
+            try:
+                node.stop(wait=False)
+                break
+            except ExecUtilException as e:
+                # it's ok to get this exception here since node
+                # could be not started yet
+                logging.info("Node is not stopped. Exception ({0}): {1}".format(type(e).__name__, e))
+            continue
+
+        logging.info("OK. Stop command was executed. Let's wait while our node will stop really.")
+        nAttempt = 0
+        while True:
+            if nAttempt == C_MAX_ATTEMPTS:
+                raise Exception("Could not stop node.")
+
+            nAttempt += 1
+            if nAttempt > 1:
+                logging.info("Wait 1 second.")
+                time.sleep(1)
+                logging.info("")
+
+            logging.info("Attempt #{0}.".format(nAttempt))
+            s1 = node.status()
+
+            if s1 == testgres.NodeStatus.Running:
+                continue
+
+            if s1 == testgres.NodeStatus.Stopped:
+                break
+
+            raise Exception("Unexpected node status: {0}.".format(s1))
+
+        logging.info("OK. Node is stopped.")
+        node.cleanup()
 
     def test_replicate(self):
         with get_new_node() as node:

tests/test_simple_remote.py

@@ -499,16 +499,61 @@ def test_backup_wrong_xlog_method(self):
                 node.backup(xlog_method='wrong')
 
     def test_pg_ctl_wait_option(self):
-        with __class__.helper__get_node() as node:
-            node.init().start(wait=False)
-            while True:
-                try:
-                    node.stop(wait=False)
-                    break
-                except ExecUtilException:
-                    # it's ok to get this exception here since node
-                    # could be not started yet
-                    pass
+        C_MAX_ATTEMPTS = 50
+
+        node = __class__.helper__get_node()
+        assert node.status() == testgres.NodeStatus.Uninitialized
+        node.init()
+        assert node.status() == testgres.NodeStatus.Stopped
+        node.start(wait=False)
+        nAttempt = 0
+        while True:
+            if nAttempt == C_MAX_ATTEMPTS:
+                raise Exception("Could not stop node.")
+
+            nAttempt += 1
+
+            if nAttempt > 1:
+                logging.info("Wait 1 second.")
+                time.sleep(1)
+                logging.info("")
+
+            logging.info("Try to stop node. Attempt #{0}.".format(nAttempt))
+
+            try:
+                node.stop(wait=False)
+                break
+            except ExecUtilException as e:
+                # it's ok to get this exception here since node
+                # could be not started yet
+                logging.info("Node is not stopped. Exception ({0}): {1}".format(type(e).__name__, e))
+            continue
+
+        logging.info("OK. Stop command was executed. Let's wait while our node will stop really.")
+        nAttempt = 0
+        while True:
+            if nAttempt == C_MAX_ATTEMPTS:
+                raise Exception("Could not stop node.")
+
+            nAttempt += 1
+            if nAttempt > 1:
+                logging.info("Wait 1 second.")
+                time.sleep(1)
+                logging.info("")
+
+            logging.info("Attempt #{0}.".format(nAttempt))
+            s1 = node.status()
+
+            if s1 == testgres.NodeStatus.Running:
+                continue
+
+            if s1 == testgres.NodeStatus.Stopped:
+                break
+
+            raise Exception("Unexpected node status: {0}.".format(s1))
+
+        logging.info("OK. Node is stopped.")
+        node.cleanup()
 
     def test_replicate(self):
         with __class__.helper__get_node() as node: