feat(test): Add MariaDB docker tasks; Add storage-related tests in GitHub workflows.

.github/workflows/unit-tests.yaml → .github/workflows/tests.yaml

@@ -17,7 +17,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  non-storage-unit-tests:
+  tests:
     strategy:
       matrix:
         os: ["ubuntu-22.04", "ubuntu-24.04"]
@@ -55,6 +55,8 @@ jobs:
           SPIDER_DEPS_MAX_PARALLELISM_PER_TASK: "1"
         run: "task deps:lib_install"
 
-      - run: "task test:cpp-non-storage-unit-tests"
+      - run: "task test:cpp-unit-tests"
 
-      - run: "task test:spider-py-non-storage-unit-tests"
+      - run: "task test:cpp-integration-tests"
+
+      - run: "task test:spider-py-unit-tests"

python/spider-py/tests/client/test_driver.py

@@ -1,5 +1,6 @@
 """Tests for the driver module."""
 
+import os
 from dataclasses import dataclass
 
 import pytest
@@ -12,7 +13,8 @@
 @pytest.fixture(scope="session")
 def driver() -> Driver:
     """Fixture for the driver."""
-    return Driver(MariaDBTestUrl)
+    url = os.getenv("SPIDER_STORAGE_URL", MariaDBTestUrl)
+    return Driver(url)
 
 
 def double(_: TaskContext, x: Int8) -> Int8:

python/spider-py/tests/storage/test_mariadb.py

@@ -1,5 +1,6 @@
 """Tests for the MariaDB storage backend."""
 
+import os
 from uuid import uuid4
 
 import msgpack
@@ -15,7 +16,8 @@
 @pytest.fixture(scope="session")
 def mariadb_storage() -> MariaDBStorage:
     """Fixture to create a MariaDB storage instance."""
-    params = parse_jdbc_url(MariaDBTestUrl)
+    url = os.getenv("SPIDER_STORAGE_URL", MariaDBTestUrl)
+    params = parse_jdbc_url(url)
     return MariaDBStorage(params)
 
 

taskfiles/lint.yaml

@@ -138,6 +138,7 @@ tasks:
     cmds:
       - for:
           - "tests/integration"
+          - "tools/scripts/storage"
         cmd: |-
           . "{{.G_LINT_VENV_DIR}}/bin/activate"
           mypy "{{.ITEM}}"

taskfiles/test.yaml

@@ -4,6 +4,9 @@ vars:
   G_UNIT_TEST_BINARY: "{{.G_BUILD_SPIDER_DIR}}/tests/unitTest"
   G_TEST_VENV_DIR: "{{.G_BUILD_DIR}}/test-venv"
   G_TEST_VENV_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/test#venv.md5"
+  G_MARIADB_DATABASE: "spider-db"
+  G_MARIADB_USERNAME: "spider-user"
+  G_MARIADB_PASSWORD: "spider-password"
 
 tasks:
   cpp-non-storage-unit-tests:
@@ -15,13 +18,49 @@ tasks:
   cpp-storage-unit-tests:
     deps:
       - "build-unit-test"
+    env:
+      SPIDER_STORAGE_URL: "jdbc:mariadb://127.0.0.1:{{.MARIADB_PORT}}/{{.G_MARIADB_DATABASE}}?user={{.G_MARIADB_USERNAME}}&password={{.G_MARIADB_PASSWORD}}"
+    vars:
+      MARIADB_CONTAINER_NAME:
+        # Normalize UUID casing: macOS generates uppercase while Linux generates lowercase.
+        sh: "uuidgen | tr '[:upper:]' '[:lower:]' | sed 's/^/spider-mariadb-/'"
+      MARIADB_PORT:
+        sh: "tools/scripts/storage/get_free_port.py"
     cmds:
+      - task: "start-storage"
+        vars:
+          MARIADB_CONTAINER_NAME: "{{.MARIADB_CONTAINER_NAME}}"
+          MARIADB_DATABASE: "{{.G_MARIADB_DATABASE}}"
+          MARIADB_USERNAME: "{{.G_MARIADB_USERNAME}}"
+          MARIADB_PASSWORD: "{{.G_MARIADB_PASSWORD}}"
+          MARIADB_PORT: "{{.MARIADB_PORT}}"
+      - defer: |-
+          {{.ROOT_DIR}}/tools/scripts/storage/stop.py \
+          --name "{{.MARIADB_CONTAINER_NAME}}"
       - "{{.G_UNIT_TEST_BINARY}} \"[storage]\""
 
   cpp-unit-tests:
     deps:
       - "build-unit-test"
+    env:
+      SPIDER_STORAGE_URL: "jdbc:mariadb://127.0.0.1:{{.MARIADB_PORT}}/{{.G_MARIADB_DATABASE}}?user={{.G_MARIADB_USERNAME}}&password={{.G_MARIADB_PASSWORD}}"
+    vars:
+      MARIADB_CONTAINER_NAME:
+        # Normalize UUID casing: macOS generates uppercase while Linux generates lowercase.
+        sh: "uuidgen | tr '[:upper:]' '[:lower:]' | sed 's/^/spider-mariadb-/'"
+      MARIADB_PORT:
+        sh: "tools/scripts/storage/get_free_port.py"
     cmds:
+      - task: "start-storage"
+        vars:
+          MARIADB_CONTAINER_NAME: "{{.MARIADB_CONTAINER_NAME}}"
+          MARIADB_DATABASE: "{{.G_MARIADB_DATABASE}}"
+          MARIADB_USERNAME: "{{.G_MARIADB_USERNAME}}"
+          MARIADB_PASSWORD: "{{.G_MARIADB_PASSWORD}}"
+          MARIADB_PORT: "{{.MARIADB_PORT}}"
+      - defer: |-
+          {{.ROOT_DIR}}/tools/scripts/storage/stop.py \
+          --name "{{.MARIADB_CONTAINER_NAME}}"
       - "{{.G_UNIT_TEST_BINARY}}"
 
   build-unit-test:
@@ -44,9 +83,28 @@ tasks:
             "spider_worker",
             "spider_scheduler",
             "integrationTest"]
-    cmd: |-
-      . ../test-venv/bin/activate
-      ../test-venv/bin/pytest tests/integration
+    env:
+      SPIDER_STORAGE_URL: "jdbc:mariadb://127.0.0.1:{{.MARIADB_PORT}}/{{.G_MARIADB_DATABASE}}?user={{.G_MARIADB_USERNAME}}&password={{.G_MARIADB_PASSWORD}}"
+    vars:
+      MARIADB_CONTAINER_NAME:
+        # Normalize UUID casing: macOS generates uppercase while Linux generates lowercase.
+        sh: "uuidgen | tr '[:upper:]' '[:lower:]' | sed 's/^/spider-mariadb-/'"
+      MARIADB_PORT:
+        sh: "tools/scripts/storage/get_free_port.py"
+    cmds:
+      - task: "start-storage"
+        vars:
+          MARIADB_CONTAINER_NAME: "{{.MARIADB_CONTAINER_NAME}}"
+          MARIADB_DATABASE: "{{.G_MARIADB_DATABASE}}"
+          MARIADB_USERNAME: "{{.G_MARIADB_USERNAME}}"
+          MARIADB_PASSWORD: "{{.G_MARIADB_PASSWORD}}"
+          MARIADB_PORT: "{{.MARIADB_PORT}}"
+      - defer: |-
+          {{.ROOT_DIR}}/tools/scripts/storage/stop.py \
+          --name "{{.MARIADB_CONTAINER_NAME}}"
+      - |-
+        . ../test-venv/bin/activate
+        ../test-venv/bin/pytest tests/integration
 
   venv:
     internal: true
@@ -82,7 +140,24 @@ tasks:
     env:
       # Don't create __pycache__ directories in the source tree.
       PYTHONDONTWRITEBYTECODE: "1"
+      SPIDER_STORAGE_URL: "jdbc:mariadb://127.0.0.1:{{.MARIADB_PORT}}/{{.G_MARIADB_DATABASE}}?user={{.G_MARIADB_USERNAME}}&password={{.G_MARIADB_PASSWORD}}"
+    vars:
+      MARIADB_CONTAINER_NAME:
+        # Normalize UUID casing: macOS generates uppercase while Linux generates lowercase.
+        sh: "uuidgen | tr '[:upper:]' '[:lower:]' | sed 's/^/spider-mariadb-/'"
+      MARIADB_PORT:
+        sh: "tools/scripts/storage/get_free_port.py"
     cmds:
+      - task: "start-storage"
+        vars:
+          MARIADB_CONTAINER_NAME: "{{.MARIADB_CONTAINER_NAME}}"
+          MARIADB_DATABASE: "{{.G_MARIADB_DATABASE}}"
+          MARIADB_USERNAME: "{{.G_MARIADB_USERNAME}}"
+          MARIADB_PASSWORD: "{{.G_MARIADB_PASSWORD}}"
+          MARIADB_PORT: "{{.MARIADB_PORT}}"
+      - defer: |-
+          {{.ROOT_DIR}}/tools/scripts/storage/stop.py \
+          --name "{{.MARIADB_CONTAINER_NAME}}"
       - "uv run pytest"
 
   spider-py-non-storage-unit-tests:
@@ -98,5 +173,48 @@ tasks:
     env:
       # Don't create __pycache__ directories in the source tree.
       PYTHONDONTWRITEBYTECODE: "1"
+      SPIDER_STORAGE_URL: "jdbc:mariadb://127.0.0.1:{{.MARIADB_PORT}}/{{.G_MARIADB_DATABASE}}?user={{.G_MARIADB_USERNAME}}&password={{.G_MARIADB_PASSWORD}}"
+    vars:
+      MARIADB_CONTAINER_NAME:
+        # Normalize UUID casing: macOS generates uppercase while Linux generates lowercase.
+        sh: "uuidgen | tr '[:upper:]' '[:lower:]' | sed 's/^/spider-mariadb-/'"
+      MARIADB_PORT:
+        sh: "tools/scripts/storage/get_free_port.py"
     cmds:
+      - task: "start-storage"
+        vars:
+          MARIADB_CONTAINER_NAME: "{{.MARIADB_CONTAINER_NAME}}"
+          MARIADB_DATABASE: "{{.G_MARIADB_DATABASE}}"
+          MARIADB_USERNAME: "{{.G_MARIADB_USERNAME}}"
+          MARIADB_PASSWORD: "{{.G_MARIADB_PASSWORD}}"
+          MARIADB_PORT: "{{.MARIADB_PORT}}"
+      - defer: |-
+          {{.ROOT_DIR}}/tools/scripts/storage/stop.py \
+          --name "{{.MARIADB_CONTAINER_NAME}}"
       - "uv run pytest -m \"storage\""
+
+  start-storage:
+    internal: true
+    vars:
+      MARIADB_CONTAINER_NAME: "{{.MARIADB_CONTAINER_NAME}}"
+      MARIADB_DATABASE: "{{.MARIADB_DATABASE}}"
+      MARIADB_USERNAME: "{{.MARIADB_USERNAME}}"
+      MARIADB_PASSWORD: "{{.MARIADB_PASSWORD}}"
+      MARIADB_PORT: "{{.MARIADB_PORT | default 3306}}"
+    cmds:
+      - |-
+        tools/scripts/storage/start.py \
+        --name "{{.MARIADB_CONTAINER_NAME}}" \
+        --port "{{.MARIADB_PORT}}" \
+        --database "{{.MARIADB_DATABASE}}" \
+        --username "{{.MARIADB_USERNAME}}" \
+        --password "{{.MARIADB_PASSWORD}}"
+      - |-
+        tools/scripts/storage/wait_for_db.py \
+        --name "{{.MARIADB_CONTAINER_NAME}}"
+      - |-
+        tools/scripts/storage/init_db.py \
+        --port "{{.MARIADB_PORT}}" \
+        --database "{{.MARIADB_DATABASE}}" \
+        --username "{{.MARIADB_USERNAME}}" \
+        --password "{{.MARIADB_PASSWORD}}"

tests/integration/client.py

@@ -1,5 +1,5 @@
 """Simple Spider client for testing purposes."""
-
+import os
 import re
 import uuid
 from collections.abc import Generator
@@ -111,7 +111,14 @@ def is_head_task(task_id: uuid.UUID, dependencies: list[tuple[uuid.UUID, uuid.UU
     return not any(dependency[1] == task_id for dependency in dependencies)
 
 
-g_storage_url = "jdbc:mariadb://localhost:3306/spider_test?user=root&password=password"
+G_STORAGE_URL = "jdbc:mariadb://localhost:3306/spider_test?user=root&password=password"
+
+def get_storage_url() -> str:
+    """
+    Gets the storage URL from the environment variable or uses the default.
+    :return: The storage URL.
+    """
+    return os.getenv("SPIDER_STORAGE_URL", G_STORAGE_URL)
 
 
 @pytest.fixture(scope="session")
@@ -122,7 +129,7 @@ def storage() -> Generator[SQLConnection, None, None]:
     after the test session is complete.
     :return: A generator yielding a MySQL connection object.
     """
-    conn = create_connection(g_storage_url)
+    conn = create_connection(get_storage_url())
     yield conn
     conn.close()
 

tests/integration/test_client.py

@@ -7,7 +7,7 @@
 
 import pytest
 
-from integration.client import g_storage_url, SQLConnection
+from integration.client import get_storage_url, SQLConnection, storage
 from integration.utils import g_scheduler_port
 
 
@@ -63,7 +63,7 @@ def scheduler_worker(
     """
     _ = storage  # Avoid ARG001
     scheduler_process, worker_process_0, worker_process_1 = start_scheduler_workers(
-        storage_url=g_storage_url, scheduler_port=g_scheduler_port
+        storage_url=get_storage_url(), scheduler_port=g_scheduler_port
     )
     # Wait for 5 second to make sure the scheduler and worker are started
     time.sleep(5)
@@ -84,7 +84,7 @@ def test_client(self) -> None:
         client_cmds = [
             str(dir_path / "client_test"),
             "--storage_url",
-            g_storage_url,
+            get_storage_url(),
         ]
         p = subprocess.run(client_cmds, check=True, timeout=20)
         assert p.returncode == 0

tests/integration/test_scheduler_worker.py

@@ -15,12 +15,13 @@
     add_driver_data,
     Data,
     Driver,
-    g_storage_url,
+    get_storage_url,
     get_task_outputs,
     get_task_state,
     remove_data,
     remove_job,
     SQLConnection,
+    storage,
     submit_job,
     Task,
     TaskGraph,
@@ -89,7 +90,7 @@ def scheduler_worker(
     """
     _ = storage  # Avoid ARG001
     scheduler_process, worker_process = start_scheduler_worker(
-        storage_url=g_storage_url, scheduler_port=g_scheduler_port
+        storage_url=get_storage_url(), scheduler_port=g_scheduler_port
     )
     # Wait for 5 second to make sure the scheduler and worker are started
     time.sleep(5)

tests/integration/test_signal.py

@@ -13,11 +13,12 @@
 import pytest
 
 from integration.client import (
-    g_storage_url,
+    get_storage_url,
     get_task_outputs,
     get_task_state,
     remove_job,
     SQLConnection,
+    storage,
     submit_job,
     Task,
     TaskGraph,
@@ -83,7 +84,7 @@ def scheduler_worker_signal(
     """
     _ = storage  # Avoid ARG001
     scheduler_process, worker_process = start_scheduler_worker(
-        storage_url=g_storage_url, scheduler_port=g_scheduler_port, lib="tests/libsignal_test.so"
+        storage_url=get_storage_url(), scheduler_port=g_scheduler_port, lib="tests/libsignal_test.so"
     )
     # Wait for 5 second to make sure the scheduler and worker are started
     time.sleep(5)

tests/storage/StorageTestHelper.hpp

@@ -5,27 +5,36 @@
 #include <concepts>
 #include <memory>
 #include <string>
+#include <string_view>
 #include <tuple>
 
+#include <boost/process/v2/environment.hpp>
+
 #include <spider/storage/mysql/MySqlStorageFactory.hpp>
 #include <spider/storage/StorageFactory.hpp>
 
 namespace spider::test {
-std::string const cMySqlStorageUrl
+constexpr std::string_view cMySqlStorageUrl
         = "jdbc:mariadb://localhost:3306/spider_test?user=root&password=password";
 
 using StorageFactoryTypeList = std::tuple<core::MySqlStorageFactory>;
 
 template <class T>
 requires std::same_as<T, core::MySqlStorageFactory>
-auto create_storage_factory() -> std::unique_ptr<core::StorageFactory> {
-    return std::make_unique<T>(cMySqlStorageUrl);
+auto get_storage_url() -> std::string {
+    auto const env = boost::process::v2::environment::current();
+    for (auto const& entry : env) {
+        if ("SPIDER_STORAGE_URL" == entry.key().string()) {
+            return entry.value().string();
+        }
+    }
+    return std::string{cMySqlStorageUrl};
 }
 
 template <class T>
 requires std::same_as<T, core::MySqlStorageFactory>
-auto get_storage_url() -> std::string {
-    return cMySqlStorageUrl;
+auto create_storage_factory() -> std::unique_ptr<core::StorageFactory> {
+    return std::make_unique<T>(get_storage_url<T>());
 }
 }  // namespace spider::test