Merge pull request #4525 from lexming/get_cwd

add new `get_cwd` function to `tools.filetools` to retrieve current working directory

Author: boegel

easybuild/base/optcomplete.py

@@ -107,6 +107,7 @@
 from optparse import OptionParser, Option
 from pprint import pformat
 
+from easybuild.tools.filetools import get_cwd
 from easybuild.tools.utilities import shell_quote
 
 debugfn = None  # for debugging only
@@ -537,7 +538,7 @@ def autocomplete(parser, arg_completer=None, opt_completer=None, subcmd_complete
         # Note: this will get filtered properly below.
 
     completer_kwargs = {
-        'pwd': os.getcwd(),
+        'pwd': get_cwd(),
         'cline': cline,
         'cpoint': cpoint,
         'prefix': prefix,

easybuild/framework/easyblock.py

@@ -81,8 +81,8 @@
 from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, check_lock
 from easybuild.tools.filetools import compute_checksum, convert_name, copy_file, create_lock, create_patch_info
 from easybuild.tools.filetools import derive_alt_pypi_url, diff_files, dir_contains_files, download_file
-from easybuild.tools.filetools import encode_class_name, extract_file
-from easybuild.tools.filetools import find_backup_name_candidate, get_source_tarball_from_git, is_alt_pypi_url
+from easybuild.tools.filetools import encode_class_name, extract_file, find_backup_name_candidate
+from easybuild.tools.filetools import get_cwd, get_source_tarball_from_git, is_alt_pypi_url
 from easybuild.tools.filetools import is_binary, is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir
 from easybuild.tools.filetools import remove_file, remove_lock, verify_checksum, weld_paths, write_file, symlink
 from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP
@@ -151,7 +151,7 @@ def __init__(self, ec):
         """
 
         # keep track of original working directory, so we can go back there
-        self.orig_workdir = os.getcwd()
+        self.orig_workdir = get_cwd()
 
         # dict of all hooks (mapping of name to function)
         self.hooks = load_hooks(build_option('hooks'))
@@ -3721,7 +3721,7 @@ def cleanup_step(self):
 
             # make sure we're out of the dir we're removing
             change_dir(self.orig_workdir)
-            self.log.info("Cleaning up builddir %s (in %s)", self.builddir, os.getcwd())
+            self.log.info("Cleaning up builddir %s (in %s)", self.builddir, get_cwd())
 
             try:
                 remove_dir(self.builddir)
@@ -4245,7 +4245,7 @@ def build_and_install_one(ecdict, init_env):
     restore_env(init_env)
     sanitize_env()
 
-    cwd = os.getcwd()
+    cwd = get_cwd()
 
     # load easyblock
     easyblock = build_option('easyblock')
@@ -4550,7 +4550,7 @@ def build_easyconfigs(easyconfigs, output_dir, test_results):
         instance = get_easyblock_instance(ec)
         apps.append(instance)
 
-    base_dir = os.getcwd()
+    base_dir = get_cwd()
 
     # keep track of environment right before initiating builds
     # note: may be different from ORIG_OS_ENVIRON, since EasyBuild may have defined additional env vars itself by now

easybuild/framework/easyconfig/tools.py

@@ -57,7 +57,7 @@
 from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning
 from easybuild.tools.config import build_option
 from easybuild.tools.environment import restore_env
-from easybuild.tools.filetools import find_easyconfigs, is_patch_file, locate_files
+from easybuild.tools.filetools import find_easyconfigs, get_cwd, is_patch_file, locate_files
 from easybuild.tools.filetools import read_file, resolve_path, which, write_file
 from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO
 from easybuild.tools.github import det_pr_labels, det_pr_title, download_repo, fetch_easyconfigs_from_commit
@@ -792,14 +792,13 @@ def det_copy_ec_specs(orig_paths, from_pr=None, from_commit=None):
 
     target_path, paths = None, []
 
-    # if only one argument is specified, use current directory as target directory
     if len(orig_paths) == 1:
-        target_path = os.getcwd()
+        # if only one argument is specified, use current directory as target directory
+        target_path = get_cwd()
         paths = orig_paths[:]
-
-    # if multiple arguments are specified, assume that last argument is target location,
-    # and remove that from list of paths to copy
     elif orig_paths:
+        # if multiple arguments are specified, assume that last argument is target location,
+        # and remove that from list of paths to copy
         target_path = orig_paths[-1]
         paths = orig_paths[:-1]
 
@@ -817,7 +816,7 @@ def det_copy_ec_specs(orig_paths, from_pr=None, from_commit=None):
             pr_paths.extend(fetch_files_from_pr(pr=pr, path=tmpdir))
 
         # assume that files need to be copied to current working directory for now
-        target_path = os.getcwd()
+        target_path = get_cwd()
 
         if orig_paths:
             last_path = orig_paths[-1]
@@ -854,7 +853,7 @@ def det_copy_ec_specs(orig_paths, from_pr=None, from_commit=None):
         commit_paths = fetch_files_from_commit(from_commit, path=tmpdir)
 
         # assume that files need to be copied to current working directory for now
-        target_path = os.getcwd()
+        target_path = get_cwd()
 
         if orig_paths:
             last_path = orig_paths[-1]

easybuild/tools/build_log.py

@@ -61,6 +61,11 @@
 DRY_RUN_SOFTWARE_INSTALL_DIR = None
 DRY_RUN_MODULES_INSTALL_DIR = None
 
+CWD_NOTFOUND_ERROR = (
+    "Current working directory does not exist! It was either unexpectedly removed "
+    "by an external process to EasyBuild or the filesystem is misbehaving."
+)
+
 
 DEVEL_LOG_LEVEL = logging.DEBUG - 1
 logging.addLevelName(DEVEL_LOG_LEVEL, 'DEVEL')

easybuild/tools/filetools.py

@@ -62,7 +62,7 @@
 
 from easybuild.base import fancylogger
 # import build_log must stay, to use of EasyBuildLog
-from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning
+from easybuild.tools.build_log import EasyBuildError, CWD_NOTFOUND_ERROR, dry_run_msg, print_msg, print_warning
 from easybuild.tools.config import ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN, build_option, install_path
 from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar
 from easybuild.tools.hooks import load_source
@@ -407,26 +407,42 @@ def remove(paths):
             raise EasyBuildError("Specified path to remove is not an existing file or directory: %s", path)
 
 
+def get_cwd(must_exist=True):
+    """
+    Retrieve current working directory
+    """
+    try:
+        cwd = os.getcwd()
+    except FileNotFoundError as err:
+        if must_exist is True:
+            raise EasyBuildError(CWD_NOTFOUND_ERROR)
+
+        _log.debug("Failed to determine current working directory, but proceeding anyway: %s", err)
+        cwd = None
+
+    return cwd
+
+
 def change_dir(path):
     """
     Change to directory at specified location.
 
     :param path: location to change to
     :return: previous location we were in
     """
-    # determining the current working directory can fail if we're in a non-existing directory
-    try:
-        cwd = os.getcwd()
-    except OSError as err:
-        _log.debug("Failed to determine current working directory (but proceeding anyway: %s", err)
-        cwd = None
+    # determine origin working directory: can fail if non-existent
+    prev_dir = get_cwd(must_exist=False)
 
     try:
         os.chdir(path)
     except OSError as err:
-        raise EasyBuildError("Failed to change from %s to %s: %s", cwd, path, err)
+        raise EasyBuildError("Failed to change from %s to %s: %s", prev_dir, path, err)
 
-    return cwd
+    # determine final working directory: must exist
+    # stoplight meant to catch filesystems in a faulty state
+    get_cwd()
+
+    return prev_dir
 
 
 def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced=False, change_into_dir=False,
@@ -671,9 +687,9 @@ def parse_http_header_fields_urlpat(arg, urlpat=None, header=None, urlpat_header
         if argline == '' or '#' in argline[0]:
             continue  # permit comment lines: ignore them
 
-        if os.path.isfile(os.path.join(os.getcwd(), argline)):
+        if os.path.isfile(os.path.join(get_cwd(), argline)):
             # expand existing relative path to absolute
-            argline = os.path.join(os.path.join(os.getcwd(), argline))
+            argline = os.path.join(os.path.join(get_cwd(), argline))
         if os.path.isfile(argline):
             # argline is a file path, so read that instead
             _log.debug('File included in parse_http_header_fields_urlpat: %s' % argline)
@@ -1332,14 +1348,14 @@ def get_local_dirs_purged():
         # and hidden directories
         ignoredirs = ["easybuild"]
 
-        lst = os.listdir(os.getcwd())
+        lst = os.listdir(get_cwd())
         lst = [d for d in lst if not d.startswith('.') and d not in ignoredirs]
         return lst
 
     lst = get_local_dirs_purged()
-    new_dir = os.getcwd()
+    new_dir = get_cwd()
     while len(lst) == 1:
-        new_dir = os.path.join(os.getcwd(), lst[0])
+        new_dir = os.path.join(get_cwd(), lst[0])
         if not os.path.isdir(new_dir):
             break
 

easybuild/tools/job/pbs_python.py

@@ -39,6 +39,7 @@
 from easybuild.base import fancylogger
 from easybuild.tools.build_log import EasyBuildError, print_msg
 from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, build_option
+from easybuild.tools.filetools import get_cwd
 from easybuild.tools.job.backend import JobBackend
 from easybuild.tools.utilities import only_if_module_is_available
 
@@ -320,8 +321,8 @@ def _submit(self):
         self.log.debug("Job hold attributes: %s" % hold_attributes[0].value)
 
         # add a bunch of variables (added by qsub)
-        # also set PBS_O_WORKDIR to os.getcwd()
-        os.environ.setdefault('WORKDIR', os.getcwd())
+        # also set PBS_O_WORKDIR to current working dir
+        os.environ.setdefault('WORKDIR', get_cwd())
 
         defvars = ['MAIL', 'HOME', 'PATH', 'SHELL', 'WORKDIR']
         pbsvars = ["PBS_O_%s=%s" % (x, os.environ.get(x, 'NOTFOUND_%s' % x)) for x in defvars]

easybuild/tools/options.py

@@ -84,8 +84,8 @@
 from easybuild.tools.docs import avail_toolchain_opts, avail_easyconfig_params, avail_easyconfig_templates
 from easybuild.tools.docs import list_easyblocks, list_toolchains
 from easybuild.tools.environment import restore_env, unset_env_vars
-from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES, expand_glob_paths, install_fake_vsc
-from easybuild.tools.filetools import move_file, which
+from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES, expand_glob_paths, get_cwd
+from easybuild.tools.filetools import install_fake_vsc, move_file, which
 from easybuild.tools.github import GITHUB_PR_DIRECTION_DESC, GITHUB_PR_ORDER_CREATED
 from easybuild.tools.github import GITHUB_PR_STATE_OPEN, GITHUB_PR_STATES, GITHUB_PR_ORDERS, GITHUB_PR_DIRECTIONS
 from easybuild.tools.github import HAVE_GITHUB_API, HAVE_KEYRING, VALID_CLOSE_PR_REASONS
@@ -829,7 +829,7 @@ def job_options(self):
             'eb-cmd': ("EasyBuild command to use in jobs", 'str', 'store', DEFAULT_JOB_EB_CMD),
             'max-jobs': ("Maximum number of concurrent jobs (queued and running, 0 = unlimited)", 'int', 'store', 0),
             'max-walltime': ("Maximum walltime for jobs (in hours)", 'int', 'store', 24),
-            'output-dir': ("Output directory for jobs (default: current directory)", None, 'store', os.getcwd()),
+            'output-dir': ("Output directory for jobs (default: current directory)", None, 'store', get_cwd()),
             'polling-interval': ("Interval between polls for status of jobs (in seconds)", float, 'store', 30.0),
             'target-resource': ("Target resource for jobs", None, 'store', None),
         })

easybuild/tools/parallelbuild.py

@@ -43,6 +43,7 @@
 from easybuild.framework.easyconfig.easyconfig import ActiveMNS
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.config import build_option, get_repository, get_repositorypath
+from easybuild.tools.filetools import get_cwd
 from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
 from easybuild.tools.job.backend import job_backend
 from easybuild.tools.repository.repository import init_repository
@@ -126,7 +127,7 @@ def submit_jobs(ordered_ecs, cmd_line_opts, testing=False, prepare_first=True):
     :param testing: If `True`, skip actual job submission
     :param prepare_first: prepare by runnning fetch step first for each easyconfig
     """
-    curdir = os.getcwd()
+    curdir = get_cwd()
 
     # regex pattern for options to ignore (help options can't reach here)
     ignore_opts = re.compile('^--robot$|^--job|^--try-.*$|^--easystack$')

easybuild/tools/repository/svnrepo.py

@@ -46,7 +46,7 @@
 
 from easybuild.base import fancylogger
 from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.filetools import remove_dir
+from easybuild.tools.filetools import get_cwd, remove_dir
 from easybuild.tools.repository.filerepo import FileRepository
 from easybuild.tools.utilities import only_if_module_is_available
 
@@ -145,7 +145,7 @@ def stage_file(self, path):
         """
         if self.client and not self.client.status(path)[0].is_versioned:
             # add it to version control
-            self.log.debug("Going to add %s (working copy: %s, cwd %s)" % (path, self.wc, os.getcwd()))
+            self.log.debug("Going to add %s (working copy: %s, cwd %s)" % (path, self.wc, get_cwd()))
             self.client.add(path)
 
     def add_easyconfig(self, cfg, name, version, stats, previous_stats):

easybuild/tools/robot.py

@@ -45,7 +45,7 @@
 from easybuild.framework.easyconfig.tools import find_resolved_modules, skip_available
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.config import build_option
-from easybuild.tools.filetools import det_common_path_prefix, search_file
+from easybuild.tools.filetools import det_common_path_prefix, get_cwd, search_file
 from easybuild.tools.module_naming_scheme.easybuild_mns import EasyBuildMNS
 from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
 from easybuild.tools.utilities import flatten, nub
@@ -491,7 +491,7 @@ def search_easyconfigs(query, short=False, filename_only=False, terse=False, con
     """
     search_path = build_option('robot_path')
     if not search_path:
-        search_path = [os.getcwd()]
+        search_path = [get_cwd()]
     extra_search_paths = build_option('search_paths')
     # If we're returning a list of possible resolutions by the robot, don't include the extra_search_paths
     if extra_search_paths and consider_extra_paths: