|
48 | 48 | import os |
49 | 49 | import re |
50 | 50 | import shutil |
| 51 | +import signal |
51 | 52 | import stat |
52 | 53 | import sys |
53 | 54 | import tempfile |
|
59 | 60 | from easybuild.tools import run |
60 | 61 | # import build_log must stay, to use of EasyBuildLog |
61 | 62 | from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning |
62 | | -from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, build_option |
| 63 | +from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, GENERIC_EASYBLOCK_PKG, build_option, install_path |
63 | 64 | from easybuild.tools.py2vs3 import std_urllib, string_type |
64 | 65 | from easybuild.tools.utilities import nub, remove_unwanted_chars |
65 | 66 |
|
|
155 | 156 | '.tar.z': "tar xzf %(filepath)s", |
156 | 157 | } |
157 | 158 |
|
| 159 | +# global set of names of locks that were created in this session |
| 160 | +global_lock_names = set() |
| 161 | + |
158 | 162 |
|
159 | 163 | class ZlibChecksum(object): |
160 | 164 | """ |
@@ -1513,6 +1517,131 @@ def mkdir(path, parents=False, set_gid=None, sticky=None): |
1513 | 1517 | _log.debug("Not creating existing path %s" % path) |
1514 | 1518 |
|
1515 | 1519 |
|
| 1520 | +def det_lock_path(lock_name): |
| 1521 | + """ |
| 1522 | + Determine full path for lock with specifed name. |
| 1523 | + """ |
| 1524 | + locks_dir = build_option('locks_dir') or os.path.join(install_path('software'), '.locks') |
| 1525 | + return os.path.join(locks_dir, lock_name + '.lock') |
| 1526 | + |
| 1527 | + |
| 1528 | +def create_lock(lock_name): |
| 1529 | + """Create lock with specified name.""" |
| 1530 | + |
| 1531 | + lock_path = det_lock_path(lock_name) |
| 1532 | + _log.info("Creating lock at %s...", lock_path) |
| 1533 | + try: |
| 1534 | + # we use a directory as a lock, since that's atomically created |
| 1535 | + mkdir(lock_path, parents=True) |
| 1536 | + global_lock_names.add(lock_name) |
| 1537 | + except EasyBuildError as err: |
| 1538 | + # clean up the error message a bit, get rid of the "Failed to create directory" part + quotes |
| 1539 | + stripped_err = str(err).split(':', 1)[1].strip().replace("'", '').replace('"', '') |
| 1540 | + raise EasyBuildError("Failed to create lock %s: %s", lock_path, stripped_err) |
| 1541 | + _log.info("Lock created: %s", lock_path) |
| 1542 | + |
| 1543 | + |
| 1544 | +def check_lock(lock_name): |
| 1545 | + """ |
| 1546 | + Check whether a lock with specified name already exists. |
| 1547 | +
|
| 1548 | + If it exists, either wait until it's released, or raise an error |
| 1549 | + (depending on --wait-on-lock configuration option). |
| 1550 | + """ |
| 1551 | + lock_path = det_lock_path(lock_name) |
| 1552 | + if os.path.exists(lock_path): |
| 1553 | + _log.info("Lock %s exists!", lock_path) |
| 1554 | + |
| 1555 | + wait_interval = build_option('wait_on_lock_interval') |
| 1556 | + wait_limit = build_option('wait_on_lock_limit') |
| 1557 | + |
| 1558 | + # --wait-on-lock is deprecated, should use --wait-on-lock-limit and --wait-on-lock-interval instead |
| 1559 | + wait_on_lock = build_option('wait_on_lock') |
| 1560 | + if wait_on_lock is not None: |
| 1561 | + depr_msg = "Use of --wait-on-lock is deprecated, use --wait-on-lock-limit and --wait-on-lock-interval" |
| 1562 | + _log.deprecated(depr_msg, '5.0') |
| 1563 | + |
| 1564 | + # if --wait-on-lock-interval has default value and --wait-on-lock is specified too, the latter wins |
| 1565 | + # (required for backwards compatibility) |
| 1566 | + if wait_interval == DEFAULT_WAIT_ON_LOCK_INTERVAL and wait_on_lock > 0: |
| 1567 | + wait_interval = wait_on_lock |
| 1568 | + |
| 1569 | + # if --wait-on-lock-limit is not specified we need to wait indefinitely if --wait-on-lock is specified, |
| 1570 | + # since the original semantics of --wait-on-lock was that it specified the waiting time interval (no limit) |
| 1571 | + if not wait_limit: |
| 1572 | + wait_limit = -1 |
| 1573 | + |
| 1574 | + # wait limit could be zero (no waiting), -1 (no waiting limit) or non-zero value (waiting limit in seconds) |
| 1575 | + if wait_limit != 0: |
| 1576 | + wait_time = 0 |
| 1577 | + while os.path.exists(lock_path) and (wait_limit == -1 or wait_time < wait_limit): |
| 1578 | + print_msg("lock %s exists, waiting %d seconds..." % (lock_path, wait_interval), |
| 1579 | + silent=build_option('silent')) |
| 1580 | + time.sleep(wait_interval) |
| 1581 | + wait_time += wait_interval |
| 1582 | + |
| 1583 | + if os.path.exists(lock_path) and wait_limit != -1 and wait_time >= wait_limit: |
| 1584 | + error_msg = "Maximum wait time for lock %s to be released reached: %s sec >= %s sec" |
| 1585 | + raise EasyBuildError(error_msg, lock_path, wait_time, wait_limit) |
| 1586 | + else: |
| 1587 | + _log.info("Lock %s was released!", lock_path) |
| 1588 | + else: |
| 1589 | + raise EasyBuildError("Lock %s already exists, aborting!", lock_path) |
| 1590 | + else: |
| 1591 | + _log.info("Lock %s does not exist", lock_path) |
| 1592 | + |
| 1593 | + |
| 1594 | +def remove_lock(lock_name): |
| 1595 | + """ |
| 1596 | + Remove lock with specified name. |
| 1597 | + """ |
| 1598 | + lock_path = det_lock_path(lock_name) |
| 1599 | + _log.info("Removing lock %s...", lock_path) |
| 1600 | + remove_dir(lock_path) |
| 1601 | + if lock_name in global_lock_names: |
| 1602 | + global_lock_names.remove(lock_name) |
| 1603 | + _log.info("Lock removed: %s", lock_path) |
| 1604 | + |
| 1605 | + |
| 1606 | +def clean_up_locks(): |
| 1607 | + """ |
| 1608 | + Clean up all still existing locks that were created in this session. |
| 1609 | + """ |
| 1610 | + for lock_name in list(global_lock_names): |
| 1611 | + remove_lock(lock_name) |
| 1612 | + |
| 1613 | + |
| 1614 | +def clean_up_locks_signal_handler(signum, frame): |
| 1615 | + """ |
| 1616 | + Signal handler, cleans up locks & exits with received signal number. |
| 1617 | + """ |
| 1618 | + |
| 1619 | + if not build_option('silent'): |
| 1620 | + print_warning("signal received (%s), cleaning up locks (%s)..." % (signum, ', '.join(global_lock_names))) |
| 1621 | + clean_up_locks() |
| 1622 | + |
| 1623 | + # by default, a KeyboardInterrupt is raised with SIGINT, so keep doing so |
| 1624 | + if signum == signal.SIGINT: |
| 1625 | + raise KeyboardInterrupt("keyboard interrupt") |
| 1626 | + else: |
| 1627 | + sys.exit(signum) |
| 1628 | + |
| 1629 | + |
| 1630 | +def register_lock_cleanup_signal_handlers(): |
| 1631 | + """ |
| 1632 | + Register signal handler for signals that cancel the current EasyBuild session, |
| 1633 | + so we can clean up the locks that were created first. |
| 1634 | + """ |
| 1635 | + signums = [ |
| 1636 | + signal.SIGABRT, |
| 1637 | + signal.SIGINT, # Ctrl-C |
| 1638 | + signal.SIGTERM, # signal 15, soft kill (like when Slurm job is cancelled or received timeout) |
| 1639 | + signal.SIGQUIT, # kinda like Ctrl-C |
| 1640 | + ] |
| 1641 | + for signum in signums: |
| 1642 | + signal.signal(signum, clean_up_locks_signal_handler) |
| 1643 | + |
| 1644 | + |
1516 | 1645 | def expand_glob_paths(glob_paths): |
1517 | 1646 | """Expand specified glob paths to a list of unique non-glob paths to only files.""" |
1518 | 1647 | paths = [] |
|
0 commit comments