diff --git a/doc/module_reference.rst b/doc/module_reference.rst index 152c15ce..2fa18010 100644 --- a/doc/module_reference.rst +++ b/doc/module_reference.rst @@ -16,6 +16,7 @@ Module Reference submodules/jenkinsbase submodules/jobs submodules/label + submodules/lockable_resources submodules/mutable_jenkins submodules/nodes submodules/plugins diff --git a/doc/submodules/lockable_resources.rst b/doc/submodules/lockable_resources.rst new file mode 100644 index 00000000..add35668 --- /dev/null +++ b/doc/submodules/lockable_resources.rst @@ -0,0 +1,7 @@ +Lockable Resources +------------------ + +.. automodule:: jenkinsapi.lockable_resources + :members: + :undoc-members: + :show-inheritance: diff --git a/jenkinsapi/jenkins.py b/jenkinsapi/jenkins.py index 2ed75b72..1e2349c2 100644 --- a/jenkinsapi/jenkins.py +++ b/jenkinsapi/jenkins.py @@ -22,6 +22,7 @@ from jenkinsapi.executors import Executors from jenkinsapi.jobs import Jobs from jenkinsapi.job import Job +from jenkinsapi.lockable_resources import LockableResources from jenkinsapi.view import View from jenkinsapi.label import Label from jenkinsapi.node import Node @@ -779,3 +780,6 @@ def http_error_302(self, req, fp, code, msg, headers): opener = build_opener(SmartRedirectHandler()) res = opener.open(request) Requester.AUTH_COOKIE = res.cookie + + def get_lockable_resources(self) -> LockableResources: + return LockableResources(self) diff --git a/jenkinsapi/lockable_resources.py b/jenkinsapi/lockable_resources.py new file mode 100644 index 00000000..6d91c285 --- /dev/null +++ b/jenkinsapi/lockable_resources.py @@ -0,0 +1,404 @@ +from abc import ABC, abstractmethod +import logging +from typing import ( + TYPE_CHECKING, + Dict, + Iterator, + List, + Mapping, + Optional, + TypedDict, +) + +from requests import Response + +from jenkinsapi.custom_exceptions import JenkinsAPIException +from jenkinsapi.jenkinsbase import JenkinsBase +from jenkinsapi.utils.retry import RetryConfig, SimpleRetryConfig + +if TYPE_CHECKING: + from jenkinsapi.jenkins import Jenkins +logger = logging.getLogger(__name__) + + +class LockableResourcePropertyDict(TypedDict): + """Property of a lockable resource, as returned by Jenkins API""" + + name: str + value: str + + +class LockableResourceDict(TypedDict, total=False): + """ + Dictionary representation of a lockable resource + + This is exactly as returned by Jenkins API + """ + + name: str + description: str + note: str + + labels: str + labelsAsList: List[str] + properties: List[LockableResourcePropertyDict] + + free: bool + stolen: bool + lockCause: Optional[str] + locked: bool + + ephemeral: bool + + reserved: bool + reservedBy: Optional[str] + reservedByEmail: Optional[str] + reservedTimestamp: Optional[int] + + buildName: Optional[str] + + +class LockableResource: + """Object representation of a lockable resource""" + + def __init__(self, parent: "LockableResources", name: str): + self.parent = parent + self.name = name + + @property + def data(self) -> LockableResourceDict: + return self.parent.data_dict[self.name] + + def is_free(self) -> bool: + """ + Check if the resource is free for reservation + + This is what the java plugin implementation checks internally + """ + return self.data["free"] + + def is_reserved(self) -> bool: + return self.data["reserved"] + + def reserve(self) -> None: + self.parent.reserve(self.name) + + def unreserve(self) -> None: + self.parent.unreserve(self.name) + + +#: Specific HTTP status code returned by API when resource is locked +HTTP_STATUS_CODE_LOCKED = 423 + + +class ResourceLockedError(JenkinsAPIException): + """Raised when a resource is locked and cannot be reserved""" + + pass + + +class ResourceReservationTimeoutError(JenkinsAPIException, TimeoutError): + """Raised when resource reservation times out""" + + pass + + +DEFAULT_WAIT_SLEEP_PERIOD = 5 +DEFAULT_WAIT_TIMEOUT_PERIOD = 3600 +DEFAULT_RETRY_CONFIG = SimpleRetryConfig( + sleep_period=DEFAULT_WAIT_SLEEP_PERIOD, + timeout=DEFAULT_WAIT_TIMEOUT_PERIOD, +) + + +class LockableResources(JenkinsBase, Mapping[str, LockableResource]): + """Object representation of the lockable resource jenkins API""" + + jenkins: "Jenkins" + + poll_after_post: bool + """ + If true then poll again after every successful post request + + This ensure that resource properties are up-to-date after any changes. + Setting this to False would require manual poll() calls but could be more + efficient in advanced scenarios with careful usage. + """ + + def __init__( + self, + jenkins_obj: "Jenkins", + poll=True, + poll_after_post: bool = True, + ): + self.jenkins = jenkins_obj + baseurl = jenkins_obj.baseurl + "/lockable-resources/api/python" + JenkinsBase.__init__(self, baseurl, poll=poll) + self.poll_after_post = poll_after_post + + def __str__(self) -> str: + return f"Lockable Resources @ {self.baseurl}" + + def get_jenkins_obj(self) -> "Jenkins": + return self.jenkins + + def poll(self, tree=None) -> None: + super().poll(tree) + self._data_dict = None + + @property + def data_list(self) -> List[LockableResourceDict]: + """API data as a list of `LockableResourceDict`""" + if self._data is None: + raise ValueError("need poll") + return self._data["resources"] + + _data_dict: Optional[Dict[str, LockableResourceDict]] = None + + @property + def data_dict(self) -> Dict[str, LockableResourceDict]: + """API data as a dict mapping name to `LockableResourceDict`""" + if self._data_dict is None: + self._data_dict = {item["name"]: item for item in self.data_list} + return self._data_dict + + def __len__(self) -> int: + return len(self.data_list) + + def __iter__(self): + return iter(self.data_dict) + + def __getitem__(self, name: str) -> LockableResource: + return LockableResource(self, name) + + def is_free(self, name: str) -> bool: + return self.data_dict[name]["free"] + + def is_reserved(self, name: str) -> bool: + return self.data_dict[name]["reserved"] + + def _make_resource_request( + self, + req: str, + name: str, + ) -> Response: + """Make a resource-specific request via HTTP POST""" + response = self.jenkins.requester.post_and_confirm_status( + self.jenkins.baseurl + f"/lockable-resources/{req}", + data=dict(resource=name), + valid=[ + 200, + HTTP_STATUS_CODE_LOCKED, + ], + ) + if response.status_code == HTTP_STATUS_CODE_LOCKED: + raise ResourceLockedError( + f"Resource {name} is busy or reserved by another user." + ) + if self.poll_after_post: + self.poll() + return response + + def reserve(self, name: str) -> None: + self._make_resource_request("reserve", name) + + def unreserve(self, name: str) -> None: + self._make_resource_request("unreserve", name) + + def try_reserve( + self, + selector: "ResourceSelector", + ) -> Optional[str]: + """ + Try to reserve a resource that matches the given condition + + :return: the name of the reserved resource on success + :return: None if all resources are busy + """ + for resource_name in selector.select(self): + resource = self[resource_name] + # if server reported that the resource is not free + # don't try to reserve it + if not resource.is_free(): + continue + # if server reported that the resource is free + # it might have been reserved since the last poll + try: + resource.reserve() + except ResourceLockedError: + continue + return resource.name + return None + + def wait_reserve( + self, + selector: "ResourceSelector", + retry: RetryConfig = DEFAULT_RETRY_CONFIG, + ) -> str: + """ + Wait for a resource that matches the given condition to become available + + :return: the name of the reserved resource on success + :raise ResourceReservationTimeoutError: if no matching resources are found during the timeout period. + """ + retry_state = retry.begin() + while True: + result = self.try_reserve(selector) + if result is not None: + return result + try: + retry_state.check_retry() + except TimeoutError as err: + raise ResourceReservationTimeoutError( + f"Timeout waiting for a resource matching {selector} after {retry}" + ) from err + logger.info("No free resources matching %r, retry", selector) + self.poll() + + def reservation_by_label( + self, + label: str, + retry: RetryConfig = DEFAULT_RETRY_CONFIG, + ) -> "LockedResourceReservation": + return LockedResourceReservation( + self, + ResourceLabelSelector(label), + retry=retry, + ) + + def reservation_by_name( + self, + name: str, + retry: RetryConfig = DEFAULT_RETRY_CONFIG, + ) -> "LockedResourceReservation": + return LockedResourceReservation( + self, + ResourceNameSelector(name), + retry=retry, + ) + + def reservation_by_name_list( + self, + name_list: List[str], + retry: RetryConfig = DEFAULT_RETRY_CONFIG, + ) -> "LockedResourceReservation": + return LockedResourceReservation( + self, + ResourceNameListSelector(name_list), + retry=retry, + ) + + +class ResourceSelector(ABC): + """Base class for which iterates acceptable resources for a reservation""" + + @abstractmethod + def select(self, lockable_resources: LockableResources) -> Iterator[str]: + """Iterate acceptable resource names""" + pass + + +class ResourceNameSelector(ResourceSelector): + """Implementation of :py:class:`ResourceSelector` that selects a single resource by name""" + + def __init__(self, name: str): + self.name = name + + def select(self, lockable_resources: LockableResources) -> Iterator[str]: + yield self.name + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.name!r})" + + +class ResourceNameListSelector(ResourceSelector): + """Implementation of :py:class:`ResourceSelector` that selects from a list of resources""" + + def __init__(self, name_list: List[str]): + self.name_list = name_list + + def select(self, lockable_resources: LockableResources) -> Iterator[str]: + return iter(self.name_list) + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.name_list!r})" + + +class ResourceLabelSelector(ResourceSelector): + """Implementation of :py:class:`ResourceSelector` that selects any resources with a given jenkins label""" + + def __init__(self, label: str): + self.label = label + + def select(self, lockable_resources: LockableResources) -> Iterator[str]: + for resource in lockable_resources.values(): + if self.label in resource.data["labelsAsList"]: + yield resource.name + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.label!r})" + + +class LockedResourceReservation: + """ + Context manager for locking a Jenkins resource + + Creating this object does not lock the resource, it is only locked and + unlocked on :meth:`__enter__` and :meth:`__exit__` methods. + + Example:: + + reservation: LockedResourceReservation = init_reservation() + # .. possibly much later ... + print("Resource will be locked ...") + with reservation as locked_resource: + name = locked_resource.locked_resource_name + print(f"Resource currently locked: {name}") + print("Resource no longer locked") + + If resources are busy this will retry until it will eventually succeed or time out. + + :raise ResourceReservationTimeoutError: if reservation process times out + """ + + _locked_resource_name: Optional[str] = None + retry: RetryConfig + + def __init__( + self, + api: LockableResources, + selector: ResourceSelector, + retry: RetryConfig = DEFAULT_RETRY_CONFIG, + ): + self.api = api + self.selector = selector + self.retry = retry + + def is_active(self) -> bool: + """Check if the resource is currently locked""" + return self._locked_resource_name is not None + + @property + def locked_resource_name(self) -> str: + """ + Return the name of the locked resource + + This throws an error if the resource is not currently locked. + """ + if self._locked_resource_name is None: + raise RuntimeError("Resource not locked") + return self._locked_resource_name + + def __enter__(self) -> "LockedResourceReservation": + """Acquire a lock for the specified label.""" + if self._locked_resource_name is not None: + raise RuntimeError("Lock already acquired") + self._locked_resource_name = self.api.wait_reserve( + self.selector, retry=self.retry + ) + return self + + def __exit__(self, *a) -> None: + if self._locked_resource_name is not None: + self.api.unreserve(self._locked_resource_name) + self._locked_resource_name = None diff --git a/jenkinsapi/utils/retry.py b/jenkinsapi/utils/retry.py new file mode 100644 index 00000000..74f623a2 --- /dev/null +++ b/jenkinsapi/utils/retry.py @@ -0,0 +1,65 @@ +import time +from abc import ABC, abstractmethod +from dataclasses import dataclass + + +class RetryConfig(ABC): + """ + Base class for retry configuration + + Usage:: + + retry_check = retry_config.begin() + while True: + result = try_something() + if result: + return result + retry_check.check_retry() + + All state is stored in the `RetryCheck` instance so `RetryConfig` can be + used in multiple contexts simultaneously. + """ + + @abstractmethod + def begin(self) -> "RetryState": ... + + +class RetryState(ABC): + """ + Base class for limited retry checks + """ + + @abstractmethod + def check_retry(self) -> None: + """Sleep or raise `TimeoutError`""" + pass + + +@dataclass +class SimpleRetryConfig(RetryConfig): + sleep_period: float = 1 + timeout: float = 5 + + def begin(self) -> "SimpleRetryState": + return SimpleRetryState(self) + + +@dataclass +class SimpleRetryState(RetryState): + """Basic implementation of RetryCheck with fixed sleep and timeout.""" + + config: SimpleRetryConfig + start_time: float + + def get_current_time(self) -> float: + return time.monotonic() + + def __init__(self, config: SimpleRetryConfig): + self.config = config + self.start_time = self.get_current_time() + + def check_retry(self) -> None: + curr_time = self.get_current_time() + if curr_time - self.start_time > self.config.timeout: + raise TimeoutError("Retry timed out") + time.sleep(self.config.sleep_period) diff --git a/jenkinsapi_tests/systests/conftest.py b/jenkinsapi_tests/systests/conftest.py index 4dbd7ed8..14253f65 100644 --- a/jenkinsapi_tests/systests/conftest.py +++ b/jenkinsapi_tests/systests/conftest.py @@ -67,6 +67,8 @@ "https://updates.jenkins.io/latest/jaxb.hpi", "https://updates.jenkins.io/latest/instance-identity.hpi", "https://updates.jenkins.io/latest/mailer.hpi", + "https://updates.jenkins.io/latest/data-tables-api.hpi", + "https://updates.jenkins.io/latest/lockable-resources.hpi", ] diff --git a/jenkinsapi_tests/systests/test_lockable_resources.py b/jenkinsapi_tests/systests/test_lockable_resources.py new file mode 100644 index 00000000..f0dbad7e --- /dev/null +++ b/jenkinsapi_tests/systests/test_lockable_resources.py @@ -0,0 +1,209 @@ +from contextlib import ExitStack +from unittest import mock +import pytest + +from jenkinsapi.jenkins import Jenkins +from jenkinsapi.lockable_resources import ( + LockableResource, + LockableResources, + ResourceLockedError, + ResourceReservationTimeoutError, +) +from jenkinsapi.utils.jenkins_launcher import JenkinsLancher +from jenkinsapi.utils.retry import SimpleRetryConfig + +GROOVY_SCRIPT_INIT_TEST_RESOURCES = """ +import org.jenkins.plugins.lockableresources.* + +def manager = LockableResourcesManager.get() + +def resource = new LockableResource("locktest") +resource.setLabels(["locktest"]) +manager.resources.add(resource) + +def resource2 = new LockableResource("locktest2") +resource2.setLabels(["locktest"]) +manager.resources.add(resource2) + +manager.save() +""" + + +@pytest.fixture(scope="module") +def jenkins_test_lock_init(launched_jenkins: JenkinsLancher) -> None: + """Fixture to create two lockable resources for testing.""" + jenkins = Jenkins(launched_jenkins.jenkins_url, timeout=30) + jenkins.run_groovy_script(GROOVY_SCRIPT_INIT_TEST_RESOURCES) + + +@pytest.fixture +def test_lock_name() -> str: + return "locktest" + + +@pytest.fixture +def test_lock_name2() -> str: + return "locktest2" + + +@pytest.fixture +def test_lock_label() -> str: + return "locktest" + + +@pytest.fixture(scope="function") +def lockable_resources( + jenkins_admin_admin: Jenkins, + jenkins_test_lock_init: None, # pylint: disable=unused-argument +) -> LockableResources: + return jenkins_admin_admin.get_lockable_resources() + + +def test_list_lockables(lockable_resources: LockableResources): + assert isinstance(str(lockable_resources), str) + assert isinstance(repr(lockable_resources), str) + + # iter names + for name in lockable_resources: + res = lockable_resources[name] + assert isinstance(res, LockableResource) + assert isinstance(res.is_free(), bool) + assert isinstance(res.is_reserved(), bool) + assert isinstance(res.data["description"], str) + # iter values directly + for res in lockable_resources.values(): + assert isinstance(res, LockableResource) + assert len(lockable_resources) == len(lockable_resources.values()) + + +def test_reserve_unreserve( + lockable_resources: LockableResources, + test_lock_name: str, +): + rn = test_lock_name + assert rn in lockable_resources + assert lockable_resources.is_reserved(rn) is False + + lockable_resources.reserve(rn) + assert lockable_resources.is_reserved(rn) is True + + with pytest.raises(ResourceLockedError): + lockable_resources.reserve(rn) + + lockable_resources.unreserve(rn) + assert lockable_resources.is_reserved(rn) is False + + +def test_reserve_unreserve_nopoll( + lockable_resources: LockableResources, + test_lock_name: str, +): + lockable_resources.poll_after_post = False + rn = test_lock_name + assert rn in lockable_resources + assert lockable_resources.is_reserved(rn) is False + + lockable_resources.reserve(rn) + assert lockable_resources.is_reserved(rn) is False + lockable_resources.poll() + assert lockable_resources.is_reserved(rn) is True + + with pytest.raises(ResourceLockedError): + lockable_resources.reserve(rn) + + lockable_resources.unreserve(rn) + assert lockable_resources.is_reserved(rn) is True + lockable_resources.poll() + assert lockable_resources.is_reserved(rn) is False + + +def test_reservation_by_name( + lockable_resources: LockableResources, + test_lock_name: str, +): + reservation = lockable_resources.reservation_by_name(test_lock_name) + assert lockable_resources.is_free(test_lock_name) + with reservation: + assert reservation.locked_resource_name == test_lock_name + assert lockable_resources.is_free(test_lock_name) is False + assert reservation.is_active() + assert lockable_resources.is_free(test_lock_name) + assert reservation.is_active() is False + name = None + with pytest.raises(RuntimeError): + name = reservation.locked_resource_name + assert name is None + + +def test_reservation_by_name_list( + lockable_resources: LockableResources, + test_lock_name: str, + test_lock_name2: str, +): + name_list = [test_lock_name, test_lock_name2] + r1 = lockable_resources.reservation_by_name_list(name_list) + assert lockable_resources.is_free(name_list[0]) + assert lockable_resources.is_free(name_list[1]) + with lockable_resources.reservation_by_name_list(name_list) as r1: + assert r1.locked_resource_name == name_list[0] + with lockable_resources.reservation_by_name_list(name_list) as r2: + assert r2.locked_resource_name == name_list[1] + assert lockable_resources.is_free(name_list[1]) is False + assert lockable_resources.is_free(name_list[1]) + assert lockable_resources.is_free(name_list[0]) + assert lockable_resources.is_free(name_list[1]) + + +def test_reservation_by_label( + lockable_resources: LockableResources, + test_lock_label: str, +): + res = lockable_resources.reservation_by_label(test_lock_label) + with res: + locked_resource = lockable_resources[res.locked_resource_name] + assert locked_resource.is_free() is False + assert test_lock_label in locked_resource.data["labelsAsList"] + assert locked_resource.is_free() is True + + +def test_custom_retry( + lockable_resources: LockableResources, + test_lock_name: str, +): + with ExitStack() as exit_stack: + exit_stack.enter_context( + mock.patch( + "time.monotonic", + side_effect=range(1000, 10000), + ) + ) + mock_time_sleep = exit_stack.enter_context( + mock.patch("time.sleep"), + ) + mock_try_reserve = exit_stack.enter_context( + mock.patch.object( + lockable_resources, + "try_reserve", + return_value=None, + ) + ) + mock_poll = exit_stack.enter_context( + mock.patch.object( + lockable_resources, + "poll", + ) + ) + exit_stack.enter_context( + pytest.raises(ResourceReservationTimeoutError) + ) + with lockable_resources.reservation_by_name( + test_lock_name, + retry=SimpleRetryConfig( + sleep_period=1, + timeout=5.5, + ), + ): + pass + assert mock_time_sleep.call_count == 5 + assert mock_try_reserve.call_count == 6 + assert mock_poll.call_count == 5 diff --git a/jenkinsapi_tests/unittests/test_retry.py b/jenkinsapi_tests/unittests/test_retry.py new file mode 100644 index 00000000..4b3c35f9 --- /dev/null +++ b/jenkinsapi_tests/unittests/test_retry.py @@ -0,0 +1,62 @@ +from contextlib import ExitStack +from unittest import mock + +import pytest + +from jenkinsapi.utils.retry import RetryConfig, RetryState, SimpleRetryConfig + + +def validate_retry_check( + retry_config: RetryConfig, + pass_index: int, + expected_sleep_count: int, + expected_pass: bool = True, +) -> None: + """Check if the retry check works as expected.""" + attempt_index = 0 + success = False + with ExitStack() as exit_stack: + exit_stack.enter_context( + mock.patch("time.monotonic", side_effect=range(100, 1000)) + ) + mock_sleep = exit_stack.enter_context(mock.patch("time.sleep")) + if not expected_pass: + exit_stack.enter_context(pytest.raises(TimeoutError)) + retry_state = retry_config.begin() + assert isinstance(retry_state, RetryState) + while True: + attempt_index += 1 + if attempt_index >= pass_index: + success = True + break + retry_state.check_retry() + if expected_pass: + assert success + else: + assert success is False + assert mock_sleep.call_count == expected_sleep_count + + +def test_simple_retry_check(): + retry_config = SimpleRetryConfig(sleep_period=1, timeout=5) + validate_retry_check( + retry_config, + pass_index=3, + expected_sleep_count=2, + expected_pass=True, + ) + + +def test_simple_retry_check_fail(): + retry_config = SimpleRetryConfig(sleep_period=1, timeout=5) + validate_retry_check( + retry_config, + pass_index=10, + expected_sleep_count=5, + expected_pass=False, + ) + + +def test_repr(): + retry_config = SimpleRetryConfig(sleep_period=1, timeout=5) + assert repr(retry_config) == "SimpleRetryConfig(sleep_period=1, timeout=5)"