Merge branch 'master' into test_delegation

Author: OlufemiAdeOlusile

.github/workflows/codeql.yml

@@ -32,7 +32,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
+      uses: github/codeql-action/init@v4
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -43,6 +43,6 @@ jobs:
         # queries: security-extended,security-and-quality
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3
+      uses: github/codeql-action/analyze@v4
       with:
         category: "/language:${{matrix.language}}"

.gitignore

@@ -100,3 +100,6 @@ ENV/
 
 # mkdocs documentation
 /site
+
+# AI helpers
+.aider*

Makefile

@@ -14,7 +14,6 @@ init_lint:
 .PHONY: lint
 lint:
 	pre-commit run -a --show-diff-on-failure --color=always
-	if command -v pytype >/dev/null 2>&1; then pytype -k -j auto cardano_node_tests; fi
 
 
 # check if development environment is set up correctly

cardano_node_tests/cluster_management/cluster_getter.py

@@ -1,4 +1,20 @@
-"""Functionality for obtaining and setting up a cluster instance."""
+"""Functionality for obtaining and setting up a cluster instance for parallel test execution.
+
+The `ClusterGetter` class is responsible for managing a pool of cluster instances and assigning them
+to tests running in parallel on different pytest workers. It ensures that tests get a suitable,
+properly configured, and healthy cluster instance to run on.
+
+Coordination between workers is achieved through a system of status files created in a shared
+temporary directory. These files signal the state of each cluster instance (e.g., running,
+needs respin), which tests are running on which instance, and what resources are locked or in use.
+
+The core logic is implemented in the `get_cluster_instance` method. It enters a loop where it
+evaluates the state of all available cluster instances against the requirements of the current test
+(e.g., resource needs, custom scripts, priority). It will wait and retry until a suitable instance
+is found and all conditions for starting the test are met. This includes handling cluster restarts
+(respins), resource allocation, and synchronization for tests that share expensive setups
+(marked tests).
+"""
 
 import dataclasses
 import logging

cardano_node_tests/cluster_management/cluster_management.py

@@ -1,4 +1,31 @@
-"""Module for exposing useful components of cluster management."""
+"""Module for exposing useful components of cluster management.
+
+The cluster management system is designed to manage a pool of Cardano cluster instances for running
+tests in parallel using `pytest-xdist`. It coordinates access to these shared cluster instances
+by multiple test workers.
+
+Key concepts:
+- **Pool of Instances**: Multiple cluster instances can be running concurrently. Each test worker
+  requests a cluster instance to run a test on.
+- **Coordination via File System**: Workers communicate and coordinate through a system of status
+  files created on a shared file system. These files act as locks and signals to indicate the
+  state of cluster instances (e.g., which test is running, if a respin is needed, which
+  resources are locked). The `status_files` module manages the creation and lookup of these
+  files.
+- **Resource Management**: Tests can declare what resources they need. A resource can be, for
+  example, a specific feature of a cluster that cannot be used by multiple tests at the same
+  time. The `ClusterManager` handles locking of these resources so that only one test can use
+  them at a time.
+- **Cluster Respin**: Some tests can modify the state of a cluster in a way that it cannot be
+  used by subsequent tests. These tests can request a "respin" of the cluster instance, which
+  re-initializes it to a clean state.
+- **`ClusterManager`**: This is the main class that test fixtures interact with. Its `get()`
+  method is used to acquire a suitable cluster instance for a test, taking into account available
+  instances, resource requirements, and scheduling priority.
+
+This system allows for efficient parallel execution of tests that require a running Cardano
+cluster, by reusing cluster instances and managing contention for shared resources.
+"""
 
 # flake8: noqa
 from cardano_node_tests.cluster_management.common import CLUSTER_LOCK

cardano_node_tests/cluster_management/manager.py

@@ -1,4 +1,13 @@
-"""Functionality for managing cluster instances."""
+"""High-level management of cluster instances.
+
+This module provides the `ClusterManager` class, which is the main interface for tests to get a
+fully initialized cluster instance. The `ClusterManager` is responsible for selecting an available
+cluster instance that meets the test's resource requirements, preparing the `clusterlib` object,
+and performing cleanup actions after the test has finished.
+
+The `ClusterManager` is instantiated by the `cluster_manager` fixture for each test worker and is
+used by the `cluster` fixture to get a cluster instance for a test.
+"""
 
 import contextlib
 import dataclasses

cardano_node_tests/cluster_management/resources_management.py

@@ -1,4 +1,17 @@
-"""Functionality for getting a cluster instance that has required resources available."""
+"""Functionality for selecting a cluster instance that has required resources available.
+
+Resources can be requested by name (string).
+
+It is also possible to use filters. A filter is a class that gets a list of all unavailable
+resources and returns a list of resources that should be used. An example is `OneOf`, which returns
+one usable resource from a given list of resources. The unavailable resources passed to the filter
+include resources that are unavailable because they are locked, and also resources that were
+already selected by preceding filters in the same request.
+
+It is possible to use multiple `OneOf` filters in a single request. For example, using `OneOf`
+filter with the same set of resources twice will result in selecting two different resources from
+that set.
+"""
 
 import random
 import typing as tp

cardano_node_tests/cluster_management/status_files.py

@@ -1,3 +1,17 @@
+"""Cluster instance status files.
+
+Status files are used for communication and synchronization between pytest workers.
+
+All status files are created in the single temp directory shared by all workers
+(the directory returned by `temptools.get_pytest_root_tmp()`). This allows all
+workers to see status files created by other workers.
+
+Common components of status file names:
+* `_@@<resource_name>@@_`: resource name
+* `_%%<mark>%%_`: test mark
+* `_<worker_id>`: pytest worker ID
+"""
+
 import pathlib as pl
 import re
 import typing as tp

cardano_node_tests/tests/tests_conway/test_drep.py

@@ -845,9 +845,11 @@ def _deregister():
 
         stake_addr_info = cluster.g_query.get_stake_addr_info(pool_user_rewards.stake.address)
 
-        assert stake_addr_info.vote_delegation == governance_utils.get_drep_cred_name(
-            drep_id=drep1.drep_id
-        ), "Votes are NOT delegated to the correct DRep 1 placed at last of certificates list."
+        assert governance_utils.get_drep_cred_name_from_addr_info(
+            addr_info=stake_addr_info
+        ) == governance_utils.get_drep_cred_name(drep_id=drep1.drep_id), (
+            "Votes are NOT delegated to the correct DRep 1 placed at last of certificates list."
+        )
         reqc.cip087.success()
 
     @allure.link(helpers.get_vcs_link())
@@ -1163,9 +1165,11 @@ def _deregister():
             f"Stake address is NOT registered: {pool_user_rewards.stake.address}"
         )
         reqc.cli035.start(url=helpers.get_vcs_link())
-        assert stake_addr_info.vote_delegation == governance_utils.get_drep_cred_name(
-            drep_id=drep_id
-        ), "Votes are NOT delegated to the correct DRep"
+        assert governance_utils.get_drep_cred_name_from_addr_info(
+            addr_info=stake_addr_info
+        ) == governance_utils.get_drep_cred_name(drep_id=drep_id), (
+            "Votes are NOT delegated to the correct DRep"
+        )
         reqc.cli035.success()
 
         out_utxos = cluster.g_query.get_utxo(tx_raw_output=tx_output)
@@ -1359,9 +1363,11 @@ def _deregister():
         stake_addr_info = cluster.g_query.get_stake_addr_info(pool_user_wpr.stake.address)
         assert stake_addr_info.delegation, f"Stake address was not delegated yet: {stake_addr_info}"
         assert pool_id == stake_addr_info.delegation, "Stake address delegated to wrong pool"
-        assert stake_addr_info.vote_delegation == governance_utils.get_drep_cred_name(
-            drep_id=drep_id
-        ), "Votes are NOT delegated to the correct DRep"
+        assert governance_utils.get_drep_cred_name_from_addr_info(
+            addr_info=stake_addr_info
+        ) == governance_utils.get_drep_cred_name(drep_id=drep_id), (
+            "Votes are NOT delegated to the correct DRep"
+        )
 
         # Check the expected balance
         out_utxos = cluster.g_query.get_utxo(tx_raw_output=tx_output)
@@ -1518,9 +1524,11 @@ def _deregister():
         stake_addr_info_deleg1 = cluster.g_query.get_stake_addr_info(
             pool_user_rewards.stake.address
         )
-        assert stake_addr_info_deleg1.vote_delegation == governance_utils.get_drep_cred_name(
-            drep_id=drep1.drep_id
-        ), "Votes are NOT delegated to the correct DRep 1"
+        assert governance_utils.get_drep_cred_name_from_addr_info(
+            addr_info=stake_addr_info_deleg1
+        ) == governance_utils.get_drep_cred_name(drep_id=drep1.drep_id), (
+            "Votes are NOT delegated to the correct DRep 1"
+        )
 
         # Change delegation to second DRep
 
@@ -1549,9 +1557,11 @@ def _deregister():
         stake_addr_info_deleg2 = cluster.g_query.get_stake_addr_info(
             pool_user_rewards.stake.address
         )
-        assert stake_addr_info_deleg2.vote_delegation == governance_utils.get_drep_cred_name(
-            drep_id=drep2.drep_id
-        ), "Votes are NOT changed to the correct DRep 2"
+        assert governance_utils.get_drep_cred_name_from_addr_info(
+            addr_info=stake_addr_info_deleg2
+        ) == governance_utils.get_drep_cred_name(drep_id=drep2.drep_id), (
+            "Votes are NOT changed to the correct DRep 2"
+        )
         reqc.cip086.success()
 
         # Retire the first DRep
@@ -1588,9 +1598,11 @@ def _deregister():
         else:
             if not stake_addr_info_ret.vote_delegation:
                 issues.ledger_4772.finish_test()
-            assert stake_addr_info_ret.vote_delegation == governance_utils.get_drep_cred_name(
-                drep_id=drep2.drep_id
-            ), "Votes are no longer delegated to DRep 2!"
+            assert governance_utils.get_drep_cred_name_from_addr_info(
+                addr_info=stake_addr_info_ret
+            ) == governance_utils.get_drep_cred_name(drep_id=drep2.drep_id), (
+                "Votes are no longer delegated to DRep 2!"
+            )
         reqc.int002.success()
 
 
@@ -1707,9 +1719,11 @@ def _delegate_addr(
             assert stake_addr_info.address, (
                 f"Stake address is NOT registered: {pool_user.stake.address}"
             )
-            assert stake_addr_info.vote_delegation == governance_utils.get_drep_cred_name(
-                drep_id=drep_reg.drep_id
-            ), "Votes are NOT delegated to the correct DRep"
+            assert governance_utils.get_drep_cred_name_from_addr_info(
+                addr_info=stake_addr_info
+            ) == governance_utils.get_drep_cred_name(drep_id=drep_reg.drep_id), (
+                "Votes are NOT delegated to the correct DRep"
+            )
 
             out_utxos = cluster.g_query.get_utxo(tx_raw_output=tx_output)
             assert (

cardano_node_tests/utils/gh_issue.py

@@ -61,7 +61,6 @@ def get_state(self) -> str | None:
         cached_state = self.issue_cache.get(identifier)
 
         if cached_state is None:
-            assert self.github  # for pytype
             try:
                 cached_state = self.github.get_repo(self.repo).get_issue(self.number).state.lower()
             except github.UnknownObjectException: