feat: add LSM and mixed UTXO backends support

- Add LSM backend support in configuration, through existing `disk`
  backend.
- Add `disklmdb` and `empty` as valid values for `UTXO_BACKEND`.
- Introduce `MIXED_UTXO_BACKENDS` environment variable for mixed setups.
- Update regression scripts and workflows to recognize new backend options.
- Document new variables and backend types in README.
- Bump `cardonnay` dependency to 0.2.11.

Author: mkoura

.github/regression.sh

@@ -77,7 +77,7 @@ fi
 
 # Decrease the number of tests per cluster if we are using the "disk" (LMDB) UTxO backend to avoid
 # having too many concurrent readers.
-if [ -z "${MAX_TESTS_PER_CLUSTER:-""}" ] && [ "${UTXO_BACKEND:-""}" = "disk" ]; then
+if [ -z "${MAX_TESTS_PER_CLUSTER:-""}" ] && [[ "${UTXO_BACKEND:-""}" = "disk"* ]]; then
   export MAX_TESTS_PER_CLUSTER=5
 fi
 

.github/workflows/regression-dbsync.yaml

@@ -51,6 +51,7 @@ on:
         - ""
         - mem
         - disk
+        - disklmdb
         default: ""
       byron_cluster:
         type: boolean

.github/workflows/regression.yaml

@@ -43,6 +43,7 @@ on:
         - ""
         - mem
         - disk
+        - disklmdb
         default: ""
       byron_cluster:
         type: boolean

README.md

@@ -114,22 +114,23 @@ For workflows requiring repeated test runs on a persistent testnet cluster:
 
 You can fine-tune test runs using these environment variables:
 
-| Variable                | Description                                |
-| ----------------------- | ------------------------------------------ |
-| `BOOTSTRAP_DIR`         | Bootstrap testnet directory.               |
-| `CLUSTERS_COUNT`        | Number of clusters to launch (default: 9). |
-| `CLUSTER_ERA`           | Cluster era (default: `conway`).           |
-| `COMMAND_ERA`           | CLI command target era.                    |
-| `ENABLE_LEGACY`         | Use legacy networking.                     |
-| `KEEP_CLUSTERS_RUNNING` | Don't shut down clusters after tests.      |
-| `MARKEXPR`              | Marker expression for pytest filtering.    |
-| `MAX_TESTS_PER_CLUSTER` | Max tests per cluster (default: 8).        |
-| `MIXED_P2P`             | Use a mix of P2P and legacy networking.    |
-| `NUM_POOLS`             | Number of stake pools (default: 3).        |
-| `PORTS_BASE`            | Starting port number for cluster services. |
-| `SCHEDULING_LOG`        | Path to scheduler log output.              |
-| `TESTNET_VARIANT`       | Name of the testnet variant to use.        |
-| `UTXO_BACKEND`          | Backend type: `mem` or `disk`.             |
+| Variable                | Description                                         |
+| ----------------------- | --------------------------------------------------- |
+| `BOOTSTRAP_DIR`         | Bootstrap testnet directory.                        |
+| `CLUSTERS_COUNT`        | Number of clusters to launch (default: 9).          |
+| `CLUSTER_ERA`           | Cluster era (default: `conway`).                    |
+| `COMMAND_ERA`           | CLI command target era.                             |
+| `ENABLE_LEGACY`         | Use legacy networking.                              |
+| `KEEP_CLUSTERS_RUNNING` | Don't shut down clusters after tests.               |
+| `MARKEXPR`              | Marker expression for pytest filtering.             |
+| `MAX_TESTS_PER_CLUSTER` | Max tests per cluster (default: 8).                 |
+| `MIXED_P2P`             | Use a mix of P2P and legacy networking.             |
+| `NUM_POOLS`             | Number of stake pools (default: 3).                 |
+| `PORTS_BASE`            | Starting port number for cluster services.          |
+| `SCHEDULING_LOG`        | Path to scheduler log output.                       |
+| `TESTNET_VARIANT`       | Name of the testnet variant to use.                 |
+| `UTXO_BACKEND`          | Backend type: `mem`, `disk`, `disklmdb` or `empty`. |
+| `MIXED_UTXO_BACKENDS`   | List of UTXO backends for mixed setup.              |
 
 ### ▶️ Additional for `regression.sh`
 

cardano_node_tests/tests/conftest.py

@@ -102,6 +102,7 @@ def pytest_configure(config: tp.Any) -> None:
     config.stash[metadata_key]["MIXED_P2P"] = str(configuration.MIXED_P2P)
     config.stash[metadata_key]["NUM_POOLS"] = str(configuration.NUM_POOLS)
     config.stash[metadata_key]["UTXO_BACKEND"] = configuration.UTXO_BACKEND
+    config.stash[metadata_key]["MIXED_UTXO_BACKENDS"] = configuration.MIXED_UTXO_BACKENDS
     config.stash[metadata_key]["MAX_TESTS_PER_CLUSTER"] = configuration.MAX_TESTS_PER_CLUSTER
     # If not explicitly specified, the `CLUSTERS_COUNT` is calculated based on number of xdist
     # workers, which is not known yet by the time this fixture runs.

cardano_node_tests/utils/configuration.py

@@ -27,9 +27,11 @@
 
 # Used also in startup scripts
 UTXO_BACKEND = os.environ.get("UTXO_BACKEND") or ""
-if UTXO_BACKEND not in ("", "mem", "disk"):
+if UTXO_BACKEND not in ("", "mem", "disk", "disklmdb", "empty"):
     msg = f"Invalid UTXO_BACKEND: {UTXO_BACKEND}"
     raise RuntimeError(msg)
+# Used also in startup scripts
+MIXED_UTXO_BACKENDS = os.environ.get("MIXED_UTXO_BACKENDS") or ""
 
 # Resolve CARDANO_NODE_SOCKET_PATH
 STARTUP_CARDANO_NODE_SOCKET_PATH = (

poetry.lock

@@ -26,7 +26,7 @@ classifiers = [
 dependencies = [
     "allure-pytest (>=2.15.2,<3.0.0)",
     "cardano-clusterlib (>=0.9.8,<0.10.0)",
-    "cardonnay (>=0.2.10,<0.3.0)",
+    "cardonnay (>=0.2.11,<0.3.0)",
     "cbor2 (>=5.7.1,<6.0.0)",
     "filelock (>=3.20.0,<4.0.0)",
     "hypothesis (>=6.148.7,<7.0.0)",