Transport: unify the behavior of core.ssh_async::glob with core.ssh::glob (#6950)

This commit suggest two changes in core.ssh_async interface:

- Catch and raise a better error message if the connection fails to open
- Globing of a non-existing path or an unmatched pattern will not raise anymore, instead it will just returns an empty list. This is to respect the same convention of core.ssh behavior

Author: khsrali

src/aiida/transports/plugins/async_backend.py

@@ -164,13 +164,16 @@ async def symlink(self, source: str, destination: str):
         """
 
     @abc.abstractmethod
-    async def glob(self, path: str):
+    async def glob(self, path: str, ignore_nonexisting: bool = True):
         """Return a list of files and directories matching the glob pattern.
         :param path: A path potentially containing the glob pattern
+        :param ignore_nonexisting: If `True` (default),
+        return an empty list when path does not exist or no matching files/folders are found.
 
         :return: A list of matching files and directories
 
-        :raises OSError: If the path does not exist or no matching files/folders are found.
+        :raises OSError: If the path does not exist or no matching files/folders are found,
+            and only if `ignore_nonexisting` is `False`.
         """
 
     @abc.abstractmethod
@@ -326,10 +329,13 @@ async def symlink(self, source: str, destination: str):
         """
         await self._sftp.symlink(source, destination)
 
-    async def glob(self, path: str):
+    async def glob(self, path: str, ignore_nonexisting: bool = True):
         try:
             return await self._sftp.glob(path)
         except asyncssh.sftp.SFTPNoSuchFile:
+            if ignore_nonexisting:
+                self.logger.debug(f'Glob pattern {path} did not match any files or directories. Ignoring.')
+                return []
             raise OSError(f'Either the remote path {path} does not exist, or a matching file/folder not found.')
 
     async def chmod(self, path: str, mode: int, follow_symlinks: bool = True):
@@ -518,11 +524,14 @@ async def chmod(self, path: str, mode: int, follow_symlinks: bool = True):
         if returncode != 0:
             raise OSError(f'Failed to change permissions: {path}')
 
-    async def glob(self, path: str):
+    async def glob(self, path: str, ignore_nonexisting: bool = True):
         commands = self.ssh_command_generator(f'find {path} -maxdepth 0')
         returncode, stdout, stderr = await self.openssh_execute(commands)
 
         if returncode != 0:
+            if ignore_nonexisting:
+                self.logger.debug(f'Glob pattern {path} did not match any files or directories. Ignoring.')
+                return []
             raise OSError(f'Either the path {path} does not exist, or a matching file/folder not found.')
 
         return list(stdout.strip().split())

src/aiida/transports/plugins/ssh_async.py

@@ -172,7 +172,11 @@ async def open_async(self):
         if self.script_before != 'None':
             os.system(f'{self.script_before}')
 
-        await self.async_backend.open()
+        try:
+            await self.async_backend.open()
+        except OSError as exc:
+            raise OSError(f'Error while opening the transport: {exc}')
+
         self._is_open = True
 
         return self
@@ -755,7 +759,7 @@ async def compress_async(
 
         for source in remotesources:
             if has_magic(source):
-                copy_list += await self.glob_async(source)
+                copy_list += await self.glob_async(source, ignore_nonexisting=False)
             else:
                 if not await self.path_exists_async(source):
                     raise OSError(f'The remote path {source} does not exist')
@@ -1161,20 +1165,21 @@ async def symlink_async(self, remotesource: TransportPath, remotedestination: Tr
         else:
             await self.async_backend.symlink(remotesource, remotedestination)
 
-    async def glob_async(self, pathname: TransportPath):
+    async def glob_async(self, pathname: TransportPath, ignore_nonexisting: bool = True):
         """Return a list of paths matching a pathname pattern.
 
         The pattern may contain simple shell-style wildcards a la fnmatch.
 
         :param pathname: the pathname pattern to match.
             It should only be absolute path.
+        :param ignore_nonexisting: if True, it will not raise and returns an empty list.
 
         :type pathname:  :class:`Path <pathlib.Path>`, :class:`PurePosixPath <pathlib.PurePosixPath>`, or `str`
 
         :return: a list of paths matching the pattern.
         """
         pathname = str(pathname)
-        return await self.async_backend.glob(pathname)
+        return await self.async_backend.glob(pathname, ignore_nonexisting=ignore_nonexisting)
 
     async def chmod_async(self, path: TransportPath, mode: int, follow_symlinks: bool = True):
         """Change the permissions of a file.

tests/transports/test_all_plugins.py

@@ -1501,6 +1501,10 @@ def test_glob(custom_transport, tmp_path_local):
         tmp_path_local.joinpath(subpath).write_text('touch')
 
     with custom_transport as transport:
+        # do not raise an error if the path does not exist
+        g_list = transport.glob(str(tmp_path_local) + '/non_existing/*.txt')
+        assert g_list == []
+
         g_list = transport.glob(str(tmp_path_local) + '/*.txt')
         paths = [str(tmp_path_local.joinpath(item)) for item in ['i.txt', 'j.txt']]
         assert sorted(paths) == sorted(g_list)