diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/tests.yaml similarity index 85% rename from .github/workflows/unit-tests.yaml rename to .github/workflows/tests.yaml index 21f3ac64..e415125a 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/tests.yaml @@ -1,4 +1,4 @@ -name: "unit-tests" +name: "tests" on: pull_request: @@ -17,7 +17,7 @@ concurrency: cancel-in-progress: true jobs: - non-storage-unit-tests: + tests: strategy: matrix: os: ["ubuntu-22.04", "ubuntu-24.04"] @@ -49,12 +49,14 @@ jobs: task --version uv --version - - name: "Install project dependencies " + - name: "Install project dependencies" timeout-minutes: 10 env: SPIDER_DEPS_MAX_PARALLELISM_PER_TASK: "1" run: "task deps:lib_install" - - run: "task test:cpp-non-storage-unit-tests" + - run: "task test:cpp-unit-tests" - - run: "task test:spider-py-non-storage-unit-tests" + - run: "task test:spider-py-unit-tests" + + - run: "task test:cpp-integration" diff --git a/docs/src/dev-docs/testing.md b/docs/src/dev-docs/testing.md index e0ddd226..efe5497e 100644 --- a/docs/src/dev-docs/testing.md +++ b/docs/src/dev-docs/testing.md @@ -1,31 +1,5 @@ # Testing -## Set up storage backend - -Spider relies on a fault-tolerant storage to store metadata and data. Spider's unit tests also -require this storage backend. - -### Set up MySQL as storage backend - -1. Start a MySQL database running in background. -2. Create an empty database. - ```sql - CREATE DATABASE ; - ``` -3. Set the password for `root` or create another user with password and grant access to database - created in step 2. - ```sql - ALTER USER 'root'@'localhost' IDENTIFIED BY ''; - --- OR create a new user - CREATE USER ''@'localhost' IDENTIFIED BY ''; - GRANT ALL PRIVILEGES ON .* TO ''@'localhost'; - ``` -4. Set the `cStorageUrl` in `tests/storage/StorageTestHelper.hpp` to - `jdbc:mariadb://localhost:3306/?user=&password=`. - -5. Set the `storage_url` in `tests/integration/client.py` to - `jdbc:mariadb://localhost:3306/?user=&password=`. - ## Running unit tests You can use the following tasks to run the set of unit tests that's appropriate. @@ -33,24 +7,8 @@ You can use the following tasks to run the set of unit tests that's appropriate. | Task | Description | |-----------------------------------------|-----------------------------------------------------------------------------| | `test:cpp-unit-tests` | Runs all C++ unit tests. | -| `test:cpp-non-storage-unit-tests` | Runs all C++ unit tests which don't require a storage backend to run. | -| `test:cpp-storage-unit-tests` | Runs all C++ unit tests which require a storage backend to run. | -| `test:spider-py-unit-tests` | Runs all spider-py unit tests. | -| `test:spider-py-non-storage-unit-tests` | Runs all spider-py unit tests which don't require a storage backend to run. | | `test:spider-py-storage-unit-tests` | Runs all spider-py unit tests which require a storage backend to run. | -If any tests show error messages for the connection function below, revisit the -[setup section](#set-up-mysql-as-storage-backend) and verify that `cStorageUrl` was set correctly. - -```c++ -REQUIRE( storage->connect(spider::test::cStorageUrl).success() ) -``` - -## GitHub unit test workflow - -The [unit_tests.yaml][gh-workflow-unit-tests] GitHub workflow runs the unit tests on push, -pull requests, and daily. Currently, it only runs unit tests that don't require a storage backend. - ## Running integration tests You can use the following tasks to run integration tests. @@ -59,5 +17,10 @@ You can use the following tasks to run integration tests. |------------------------|---------------------------------| | `test:cpp-integration` | Runs all C++ integration tests. | +## GitHub test workflow + +The [tests.yaml][gh-workflow-tests] GitHub workflow runs all unit tests and integration tests on +push, pull requests, and daily. + -[gh-workflow-unit-tests]: https://github.com/y-scope/spider/blob/main/.github/workflows/unit-tests.yaml +[gh-workflow-tests]: https://github.com/y-scope/spider/blob/main/.github/workflows/tests.yaml diff --git a/python/spider-py/README.md b/python/spider-py/README.md index 0a6bc871..ce86ba20 100644 --- a/python/spider-py/README.md +++ b/python/spider-py/README.md @@ -26,60 +26,12 @@ directory at the Spider project root. ## Testing -Unit tests are divided into two categories: storage and non-storage tests. Non-storage tests do not -require any external services, while storage tests require a MariaDB instance to be available. - -### Non-Storage Unit Tests - -To run all non-storage unit tests: - -```shell -task test:spider-py-non-storage-unit-tests -``` - -### Setup MariaDB for Storage Unit Tests - -To run storage unit tests, we need to create a MariaDB instance first. - -```shell -docker run \ - --detach \ - --rm \ - --name spider-storage \ - --env MARIADB_USER=spider \ - --env MARIADB_PASSWORD=password \ - --env MARIADB_DATABASE=spider-storage \ - --env MARIADB_ALLOW_EMPTY_ROOT_PASSWORD=true \ - --publish 3306:3306 mariadb:latest -``` - -After the docker container starts, set up the database table manually by using the SQL script -`tools/scripts/storage/init_db.sql` from the project root. - -```shell -mysql -h 127.0.0.1 -u spider -ppassword spider-storage < tools/scripts/storage/init_db.sql -``` - -### Storage Unit Tests - -To run all storage unit tests: - -```shell -task test:spider-py-storage-unit-tests -``` - -This requires a running MariaDB instance as described above. - -### All Unit Tests - -To run all unit tests (both storage and non-storage): +To run all unit tests: ```shell task test:spider-py-unit-tests ``` -This requires a running MariaDB instance as described above. - ## Linting To run all linting checks: diff --git a/python/spider-py/tests/client/test_driver.py b/python/spider-py/tests/client/test_driver.py index 484bb259..921e9dbf 100644 --- a/python/spider-py/tests/client/test_driver.py +++ b/python/spider-py/tests/client/test_driver.py @@ -1,5 +1,6 @@ """Tests for the driver module.""" +import os from dataclasses import dataclass import pytest @@ -12,7 +13,8 @@ @pytest.fixture(scope="session") def driver() -> Driver: """Fixture for the driver.""" - return Driver(MariaDBTestUrl) + url = os.getenv("SPIDER_STORAGE_URL", MariaDBTestUrl) + return Driver(url) def double(_: TaskContext, x: Int8) -> Int8: diff --git a/python/spider-py/tests/storage/test_mariadb.py b/python/spider-py/tests/storage/test_mariadb.py index a2e97ec5..10f77aee 100644 --- a/python/spider-py/tests/storage/test_mariadb.py +++ b/python/spider-py/tests/storage/test_mariadb.py @@ -1,5 +1,6 @@ """Tests for the MariaDB storage backend.""" +import os from uuid import uuid4 import msgpack @@ -15,7 +16,8 @@ @pytest.fixture(scope="session") def mariadb_storage() -> MariaDBStorage: """Fixture to create a MariaDB storage instance.""" - params = parse_jdbc_url(MariaDBTestUrl) + url = os.getenv("SPIDER_STORAGE_URL", MariaDBTestUrl) + params = parse_jdbc_url(url) return MariaDBStorage(params) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 2bc30abb..596377b9 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -138,6 +138,7 @@ tasks: cmds: - for: - "tests/integration" + - "tools/scripts/mariadb" cmd: |- . "{{.G_LINT_VENV_DIR}}/bin/activate" mypy "{{.ITEM}}" diff --git a/taskfiles/test.yaml b/taskfiles/test.yaml index b9b73317..bd88cf5c 100644 --- a/taskfiles/test.yaml +++ b/taskfiles/test.yaml @@ -5,24 +5,29 @@ vars: G_TEST_VENV_DIR: "{{.G_BUILD_DIR}}/test-venv" G_TEST_VENV_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/test#venv.md5" + # MariaDB testing config + G_MARIADB_DATABASE: "spider-db" + G_MARIADB_USERNAME: "spider-user" + G_MARIADB_PASSWORD: "spider-password" + tasks: - cpp-non-storage-unit-tests: - deps: - - "build-unit-test" + cpp-unit-tests: cmds: - - "{{.G_UNIT_TEST_BINARY}} \"~[storage]\"" + - task: "mariadb-storage-task-executor" + vars: + STORAGE_TASK: "spider-wolf-unit-tests-executor" - cpp-storage-unit-tests: - deps: - - "build-unit-test" + cpp-integration: cmds: - - "{{.G_UNIT_TEST_BINARY}} \"[storage]\"" + - task: "mariadb-storage-task-executor" + vars: + STORAGE_TASK: "spider-wolf-integration-tests-executor" - cpp-unit-tests: - deps: - - "build-unit-test" + spider-py-unit-tests: cmds: - - "{{.G_UNIT_TEST_BINARY}}" + - task: "mariadb-storage-task-executor" + vars: + STORAGE_TASK: "spider-py-unit-tests-executor" build-unit-test: internal: true @@ -31,23 +36,6 @@ tasks: vars: TARGETS: ["spider_task_executor", "unitTest", "worker_test"] - cpp-integration: - dir: "{{.G_BUILD_SPIDER_DIR}}" - deps: - - "venv" - - task: ":build:cpp-target" - vars: - TARGETS: [ - "spider_task_executor", - "worker_test", - "client_test", - "spider_worker", - "spider_scheduler", - "integrationTest"] - cmd: |- - . ../test-venv/bin/activate - ../test-venv/bin/pytest tests/integration - venv: internal: true vars: @@ -77,26 +65,95 @@ tasks: CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] - spider-py-unit-tests: - dir: "{{.G_SRC_PYTHON_DIR}}" - env: - # Don't create __pycache__ directories in the source tree. - PYTHONDONTWRITEBYTECODE: "1" + # A generic wrapper that runs the given task with a MariaDB storage backend. + # + # @param {string} STORAGE_TASK The task to execute. The task must accept no parameters other than + # `SPIDER_STORAGE_URL`, which is set to the MariaDB instance URL. + mariadb-storage-task-executor: + internal: true + vars: + MARIADB_CONTAINER_NAME: + # Normalize UUID casing: macOS generates uppercase while Linux generates lowercase. + sh: "uuidgen | tr '[:upper:]' '[:lower:]' | sed 's/^/spider-mariadb-/'" + MARIADB_PORT: + sh: "tools/scripts/get_free_port.py" + requires: + vars: ["STORAGE_TASK"] + dir: "{{.ROOT_DIR}}" cmds: - - "uv run pytest" + - |- + tools/scripts/mariadb/start.py \ + --name "{{.MARIADB_CONTAINER_NAME}}" \ + --port "{{.MARIADB_PORT}}" \ + --database "{{.G_MARIADB_DATABASE}}" \ + --username "{{.G_MARIADB_USERNAME}}" \ + --password "{{.G_MARIADB_PASSWORD}}" + - defer: |- + {{.ROOT_DIR}}/tools/scripts/mariadb/stop.py \ + --name "{{.MARIADB_CONTAINER_NAME}}" + - |- + tools/scripts/mariadb/wolf/init_db.py \ + --port "{{.MARIADB_PORT}}" \ + --database "{{.G_MARIADB_DATABASE}}" \ + --username "{{.G_MARIADB_USERNAME}}" \ + --password "{{.G_MARIADB_PASSWORD}}" + - task: "{{.STORAGE_TASK}}" + vars: + SPIDER_STORAGE_URL: + "jdbc:mariadb://127.0.0.1:{{.MARIADB_PORT}}/{{.G_MARIADB_DATABASE}}?\ + user={{.G_MARIADB_USERNAME}}&password={{.G_MARIADB_PASSWORD}}" - spider-py-non-storage-unit-tests: - dir: "{{.G_SRC_PYTHON_DIR}}" + # Internal task that runs all spider-py's unit tests. + # + # @param {string} SPIDER_STORAGE_URL An URL pointing to the MariaDB instance. + spider-py-unit-tests-executor: + internal: true env: # Don't create __pycache__ directories in the source tree. PYTHONDONTWRITEBYTECODE: "1" - cmds: - - "uv run pytest -m \"not storage\"" - - spider-py-storage-unit-tests: + SPIDER_STORAGE_URL: "{{.SPIDER_STORAGE_URL}}" + requires: + vars: ["SPIDER_STORAGE_URL"] dir: "{{.G_SRC_PYTHON_DIR}}" + cmd: "uv run pytest" + + # Internal task that runs all Spider Wolf's unit tests. + # + # @param {string} SPIDER_STORAGE_URL An URL pointing to the MariaDB instance. + spider-wolf-unit-tests-executor: + internal: true env: - # Don't create __pycache__ directories in the source tree. - PYTHONDONTWRITEBYTECODE: "1" + SPIDER_STORAGE_URL: "{{.SPIDER_STORAGE_URL}}" + requires: + vars: ["SPIDER_STORAGE_URL"] + dir: "{{.ROOT_DIR}}" + deps: + - "build-unit-test" + cmd: "{{.G_UNIT_TEST_BINARY}}" + + # Internal task that runs all Spider Wolf's integration tests. + # + # @param {string} SPIDER_STORAGE_URL An URL pointing to the MariaDB instance. + spider-wolf-integration-tests-executor: + internal: true + env: + SPIDER_STORAGE_URL: "{{.SPIDER_STORAGE_URL}}" + requires: + vars: ["SPIDER_STORAGE_URL"] + dir: "{{.G_BUILD_SPIDER_DIR}}" + deps: + - "venv" + - task: ":build:cpp-target" + vars: + TARGETS: [ + "spider_task_executor", + "worker_test", + "client_test", + "spider_worker", + "spider_scheduler", + "integrationTest" + ] cmds: - - "uv run pytest -m \"storage\"" + - |- + . {{.G_TEST_VENV_DIR}}/bin/activate + {{.G_TEST_VENV_DIR}}/bin/pytest tests/integration diff --git a/tests/integration/client.py b/tests/integration/client.py index 05a3fa5c..9d1162d0 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -1,5 +1,6 @@ """Simple Spider client for testing purposes.""" +import os import re import uuid from collections.abc import Generator @@ -111,7 +112,15 @@ def is_head_task(task_id: uuid.UUID, dependencies: list[tuple[uuid.UUID, uuid.UU return not any(dependency[1] == task_id for dependency in dependencies) -g_storage_url = "jdbc:mariadb://localhost:3306/spider_test?user=root&password=password" +G_STORAGE_URL = "jdbc:mariadb://localhost:3306/spider_test?user=root&password=password" + + +def get_storage_url() -> str: + """ + Gets the storage URL from the environment variable or uses the default. + :return: The storage URL. + """ + return os.getenv("SPIDER_STORAGE_URL", G_STORAGE_URL) @pytest.fixture(scope="session") @@ -122,7 +131,7 @@ def storage() -> Generator[SQLConnection, None, None]: after the test session is complete. :return: A generator yielding a MySQL connection object. """ - conn = create_connection(g_storage_url) + conn = create_connection(get_storage_url()) yield conn conn.close() diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index 3d01a335..4d8b4700 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -7,7 +7,7 @@ import pytest -from integration.client import g_storage_url, SQLConnection +from integration.client import get_storage_url, SQLConnection, storage # noqa: F401 from integration.utils import g_scheduler_port @@ -52,7 +52,7 @@ def start_scheduler_workers( @pytest.fixture(scope="class") def scheduler_worker( - storage: SQLConnection, + storage: SQLConnection, # noqa: F811 ) -> Generator[None, None, None]: """ Fixture to start a scheduler process and two worker processes. @@ -63,7 +63,7 @@ def scheduler_worker( """ _ = storage # Avoid ARG001 scheduler_process, worker_process_0, worker_process_1 = start_scheduler_workers( - storage_url=g_storage_url, scheduler_port=g_scheduler_port + storage_url=get_storage_url(), scheduler_port=g_scheduler_port ) # Wait for 5 second to make sure the scheduler and worker are started time.sleep(5) @@ -84,7 +84,7 @@ def test_client(self) -> None: client_cmds = [ str(dir_path / "client_test"), "--storage_url", - g_storage_url, + get_storage_url(), ] p = subprocess.run(client_cmds, check=True, timeout=20) assert p.returncode == 0 diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 060e3e3f..8e06da2e 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -15,12 +15,13 @@ add_driver_data, Data, Driver, - g_storage_url, + get_storage_url, get_task_outputs, get_task_state, remove_data, remove_job, SQLConnection, + storage, # noqa: F401 submit_job, Task, TaskGraph, @@ -78,7 +79,7 @@ def start_scheduler_worker( @pytest.fixture(scope="class") def scheduler_worker( - storage: SQLConnection, + storage: SQLConnection, # noqa: F811 ) -> Generator[None, None, None]: """ Fixture to start qa scheduler process and a worker processes. @@ -89,7 +90,7 @@ def scheduler_worker( """ _ = storage # Avoid ARG001 scheduler_process, worker_process = start_scheduler_worker( - storage_url=g_storage_url, scheduler_port=g_scheduler_port + storage_url=get_storage_url(), scheduler_port=g_scheduler_port ) # Wait for 5 second to make sure the scheduler and worker are started time.sleep(5) @@ -100,7 +101,7 @@ def scheduler_worker( @pytest.fixture def success_job( - storage: SQLConnection, + storage: SQLConnection, # noqa: F811 ) -> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: """ Fixture to create a job with two parent tasks and one child task. @@ -167,7 +168,7 @@ def success_job( @pytest.fixture def fail_job( - storage: SQLConnection, + storage: SQLConnection, # noqa: F811 ) -> Generator[Task, None, None]: """ Fixture to create a job that will fail. The task will raise an error when executed. @@ -197,7 +198,7 @@ def fail_job( @pytest.fixture def data_job( - storage: SQLConnection, + storage: SQLConnection, # noqa: F811 ) -> Generator[Task, None, None]: """ Fixture to create a data and a task that uses the data. @@ -236,7 +237,7 @@ def data_job( @pytest.fixture def random_fail_job( - storage: SQLConnection, + storage: SQLConnection, # noqa: F811 ) -> Generator[Task, None, None]: """ Fixture to create a job that randomly fails. The task will succeed after a few retries. @@ -280,7 +281,7 @@ class TestSchedulerWorker: @pytest.mark.usefixtures("scheduler_worker") def test_job_success( self, - storage: SQLConnection, + storage: SQLConnection, # noqa: F811 success_job: tuple[TaskGraph, Task, Task, Task], ) -> None: """ @@ -310,7 +311,7 @@ def test_job_success( @pytest.mark.usefixtures("scheduler_worker") def test_job_failure( self, - storage: SQLConnection, + storage: SQLConnection, # noqa: F811 fail_job: Task, ) -> None: """ @@ -327,7 +328,7 @@ def test_job_failure( @pytest.mark.usefixtures("scheduler_worker") def test_data_job( self, - storage: SQLConnection, + storage: SQLConnection, # noqa: F811 data_job: Task, ) -> None: """ @@ -347,7 +348,7 @@ def test_data_job( @pytest.mark.usefixtures("scheduler_worker") def test_random_fail_job( self, - storage: SQLConnection, + storage: SQLConnection, # noqa: F811 random_fail_job: Task, ) -> None: """ diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index ece33a74..ed450c77 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -13,11 +13,12 @@ import pytest from integration.client import ( - g_storage_url, + get_storage_url, get_task_outputs, get_task_state, remove_job, SQLConnection, + storage, # noqa: F401 submit_job, Task, TaskGraph, @@ -72,7 +73,7 @@ def start_scheduler_worker( @pytest.fixture def scheduler_worker_signal( - storage: SQLConnection, + storage: SQLConnection, # noqa: F811 ) -> Generator[tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]], None, None]: """ Fixture to start a scheduler and a worker process. @@ -83,7 +84,9 @@ def scheduler_worker_signal( """ _ = storage # Avoid ARG001 scheduler_process, worker_process = start_scheduler_worker( - storage_url=g_storage_url, scheduler_port=g_scheduler_port, lib="tests/libsignal_test.so" + storage_url=get_storage_url(), + scheduler_port=g_scheduler_port, + lib="tests/libsignal_test.so", ) # Wait for 5 second to make sure the scheduler and worker are started time.sleep(5) @@ -97,7 +100,7 @@ class TestWorkerSignal: def test_task_signal( self, - storage: SQLConnection, + storage: SQLConnection, # noqa: F811 scheduler_worker_signal: tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]], ) -> None: """ @@ -174,7 +177,7 @@ def test_task_signal( def test_task_exit( self, - storage: SQLConnection, + storage: SQLConnection, # noqa: F811 scheduler_worker_signal: tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]], ) -> None: """ diff --git a/tests/storage/StorageTestHelper.hpp b/tests/storage/StorageTestHelper.hpp index 737f47c2..b1e48ae6 100644 --- a/tests/storage/StorageTestHelper.hpp +++ b/tests/storage/StorageTestHelper.hpp @@ -5,27 +5,36 @@ #include #include #include +#include #include +#include + #include #include namespace spider::test { -std::string const cMySqlStorageUrl +constexpr std::string_view cMySqlStorageUrl = "jdbc:mariadb://localhost:3306/spider_test?user=root&password=password"; using StorageFactoryTypeList = std::tuple; template requires std::same_as -auto create_storage_factory() -> std::unique_ptr { - return std::make_unique(cMySqlStorageUrl); +auto get_storage_url() -> std::string { + auto const env = boost::process::v2::environment::current(); + for (auto const& entry : env) { + if ("SPIDER_STORAGE_URL" == entry.key().string()) { + return entry.value().string(); + } + } + return std::string{cMySqlStorageUrl}; } template requires std::same_as -auto get_storage_url() -> std::string { - return cMySqlStorageUrl; +auto create_storage_factory() -> std::unique_ptr { + return std::make_unique(get_storage_url()); } } // namespace spider::test diff --git a/tools/scripts/get_free_port.py b/tools/scripts/get_free_port.py new file mode 100755 index 00000000..188c134b --- /dev/null +++ b/tools/scripts/get_free_port.py @@ -0,0 +1,30 @@ +#!/usr/bin/env -S uv run --script +# /// script +# dependencies = [] +# /// +"""Script to get an unused port and print the port number to stdout.""" + +import socket +import sys + + +def get_free_port() -> int: + """ + Finds an unused port on localhost. + :return: An unused port number. + """ + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("", 0)) + return int(s.getsockname()[1]) + + +def main() -> int: + """Main.""" + port = get_free_port() + # ruff: noqa: T201 + print(port) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/scripts/mariadb/start.py b/tools/scripts/mariadb/start.py new file mode 100755 index 00000000..9dc0391d --- /dev/null +++ b/tools/scripts/mariadb/start.py @@ -0,0 +1,164 @@ +#!/usr/bin/env -S uv run --script +# /// script +# dependencies = [] +# /// +"""Script to start a MariaDB Docker container.""" + +import argparse +import logging +import subprocess +import sys +import time + +_MARIADB_IMAGE = "mariadb:latest" + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", +) +logger = logging.getLogger(__name__) + + +def main() -> int: + """Main.""" + parser = argparse.ArgumentParser(description="Start MariaDB Docker container.") + parser.add_argument( + "--name", + type=str, + default="mariadb-spider-dev", + help="The name of the started MariaDB Docker container (default: %(default)s)", + ) + parser.add_argument( + "--port", + type=int, + default=3306, + help="The port to expose MariaDB on (default: %(default)d)", + ) + parser.add_argument( + "--username", + type=str, + default="spider-user", + help="The username of the started MariaDB (default: %(default)s)", + ) + parser.add_argument( + "--password", + type=str, + default="spider-password", + help="The password of the started MariaDB (default: %(default)s)", + ) + parser.add_argument( + "--database", + type=str, + default="spider-db", + help="The database name of the started MariaDB (default: %(default)s)", + ) + parser.add_argument( + "--timeout", + type=int, + default=120, + help="The timeout in seconds to wait for the container to be ready (default: %(default)s)", + ) + args = parser.parse_args() + + # Silence Ruff S607: the absolute path of the Docker binary may vary depending on the + # installation method. + docker_executable = "docker" + + result = subprocess.run( + [docker_executable, "inspect", "-f", "{{.State.Running}}", args.name], + capture_output=True, + text=True, + check=False, + ) + if result.returncode == 0 and result.stdout.rstrip("\n") == "true": + logger.info("Container %s already exists.", args.name) + return 1 + + logger.info("Starting MariaDB container %s on port %d.", args.name, args.port) + logger.info("Pulling latest Mariadb image.") + result = subprocess.run( + [docker_executable, "pull", _MARIADB_IMAGE], + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + logger.error("Failed to pull MariaDB image:\n%s", result.stderr) + return result.returncode + logger.info("Successfully pulled latest MariaDB image.") + + mariadb_start_cmd = [ + docker_executable, + "run", + "--rm", + "-d", + "--name", + args.name, + "-e", + f"MARIADB_USER={args.username}", + "-e", + f"MARIADB_PASSWORD={args.password}", + "-e", + f"MARIADB_DATABASE={args.database}", + "-e", + f"MARIADB_ROOT_PASSWORD={args.password}", + "-p", + f"{args.port}:3306", + _MARIADB_IMAGE, + ] + + result = subprocess.run( + mariadb_start_cmd, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + logger.error("Failed to start MariaDB container:\n%s", result.stderr) + return result.returncode + logger.info("MariaDB container started successfully with ID: %s.", result.stdout.strip()) + + start = time.time() + while True: + result = subprocess.run( + [ + docker_executable, + "exec", + args.name, + "healthcheck.sh", + "--connect", + "--innodb_initialized", + ], + capture_output=True, + text=True, + check=False, + ) + if result.returncode == 0: + logger.info("MariaDB container '%s' is ready for connections.", args.name) + return 0 + if time.time() - start > args.timeout: + break + + time.sleep(5) + + # Cleanup on failure + logger.error("Timeout reached. MariaDB container '%s' is not ready.", args.name) + logger.info("Stopping MariaDB container '%s'.", args.name) + mariadb_stop_cmd = [ + docker_executable, + "stop", + args.name, + ] + # Ignore any failures in the stop command + _ = subprocess.run( + mariadb_stop_cmd, + capture_output=True, + text=True, + check=False, + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/scripts/mariadb/stop.py b/tools/scripts/mariadb/stop.py new file mode 100755 index 00000000..eec75f85 --- /dev/null +++ b/tools/scripts/mariadb/stop.py @@ -0,0 +1,59 @@ +#!/usr/bin/env -S uv run --script +# /// script +# dependencies = [] +# /// +"""Script to stop a running MariaDB Docker container.""" + +import argparse +import logging +import subprocess +import sys + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", +) +logger = logging.getLogger(__name__) + + +def main() -> int: + """Main.""" + # To silence Ruff S607 + docker_executable = "docker" + + parser = argparse.ArgumentParser(description="Stop MariaDB Docker container.") + parser.add_argument( + "--name", + type=str, + default="mariadb-spider-dev", + help="The name of the started MariaDB container (default: %(default)s)", + ) + args = parser.parse_args() + + result = subprocess.run( + [docker_executable, "inspect", "-f", "{{.State.Running}}", args.name], + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0 or result.stdout.rstrip("\n") != "true": + logger.warning("Container '%s' doesn't exist. Exit peacefully.", args.name) + return 0 + + mariadb_stop_cmd = [ + docker_executable, + "stop", + args.name, + ] + + result = subprocess.run(mariadb_stop_cmd, capture_output=True, text=True, check=False) + if result.returncode != 0: + logger.error("Failed to stop MariaDB container:\n%s", result.stderr) + return result.returncode + logger.info("MariaDB container stopped successfully.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/scripts/mariadb/wolf/init_db.py b/tools/scripts/mariadb/wolf/init_db.py new file mode 100755 index 00000000..805ba0db --- /dev/null +++ b/tools/scripts/mariadb/wolf/init_db.py @@ -0,0 +1,272 @@ +#!/usr/bin/env -S uv run --script +# /// script +# dependencies = [ +# "mariadb>=1.1.14", +# ] +# /// +"""Script to initialize database tables for Spider.""" + +import argparse +import logging +import sys + +import mariadb # type: ignore [import-not-found] + +_TABLE_CREATORS = [ + """ + CREATE TABLE IF NOT EXISTS `drivers` ( + `id` BINARY(16) NOT NULL, + `heartbeat` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) + ); + """, + """ + CREATE TABLE IF NOT EXISTS `schedulers` ( + `id` BINARY(16) NOT NULL, + `address` VARCHAR(40) NOT NULL, + `port` INT UNSIGNED NOT NULL, + CONSTRAINT `scheduler_driver_id` FOREIGN KEY (`id`) REFERENCES `drivers` (`id`) + ON UPDATE NO ACTION ON DELETE CASCADE, + PRIMARY KEY (`id`) + ); + """, + """ + CREATE TABLE IF NOT EXISTS jobs ( + `id` BINARY(16) NOT NULL, + `client_id` BINARY(16) NOT NULL, + `creation_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + `state` ENUM ('running', 'success', 'fail', 'cancel') NOT NULL DEFAULT 'running', + KEY (`client_id`) USING BTREE, + INDEX idx_jobs_creation_time (`creation_time`), + INDEX idx_jobs_state (`state`), + PRIMARY KEY (`id`) + ); + """, + """ + CREATE TABLE IF NOT EXISTS tasks ( + `id` BINARY(16) NOT NULL, + `job_id` BINARY(16) NOT NULL, + `func_name` VARCHAR(64) NOT NULL, + `language` ENUM ('cpp', 'python') NOT NULL, + `state` ENUM ('pending', 'ready', 'running', 'success', 'cancel', 'fail') NOT NULL, + `timeout` FLOAT, + `max_retry` INT UNSIGNED DEFAULT 0, + `retry` INT UNSIGNED DEFAULT 0, + `instance_id` BINARY(16), + CONSTRAINT `task_job_id` FOREIGN KEY (`job_id`) REFERENCES `jobs` (`id`) + ON UPDATE NO ACTION ON DELETE CASCADE, + PRIMARY KEY (`id`) + ); + """, + """ + CREATE TABLE IF NOT EXISTS input_tasks ( + `job_id` BINARY(16) NOT NULL, + `task_id` BINARY(16) NOT NULL, + `position` INT UNSIGNED NOT NULL, + CONSTRAINT `input_task_job_id` FOREIGN KEY (`job_id`) REFERENCES `jobs` (`id`) + ON UPDATE NO ACTION ON DELETE CASCADE, + CONSTRAINT `input_task_task_id` FOREIGN KEY (`task_id`) REFERENCES `tasks` (`id`) + ON UPDATE NO ACTION ON DELETE CASCADE, + INDEX (`job_id`, `position`), + PRIMARY KEY (`task_id`) + ); + """, + """ + CREATE TABLE IF NOT EXISTS output_tasks ( + `job_id` BINARY(16) NOT NULL, + `task_id` BINARY(16) NOT NULL, + `position` INT UNSIGNED NOT NULL, + CONSTRAINT `output_task_job_id` FOREIGN KEY (`job_id`) REFERENCES `jobs` (`id`) + ON UPDATE NO ACTION ON DELETE CASCADE, + CONSTRAINT `output_task_task_id` FOREIGN KEY (`task_id`) REFERENCES `tasks` (`id`) + ON UPDATE NO ACTION ON DELETE CASCADE, + INDEX (`job_id`, `position`), + PRIMARY KEY (`task_id`) + ); + """, + """ + CREATE TABLE IF NOT EXISTS `data` ( + `id` BINARY(16) NOT NULL, + `value` VARBINARY(999) NOT NULL, + `hard_locality` BOOL DEFAULT FALSE, + `persisted` BOOL DEFAULT FALSE, + PRIMARY KEY (`id`) + ); + """, + """ + CREATE TABLE IF NOT EXISTS `task_outputs` ( + `task_id` BINARY(16) NOT NULL, + `position` INT UNSIGNED NOT NULL, + `type` VARCHAR(999) NOT NULL, + `value` VARBINARY(999), + `data_id` BINARY(16), + CONSTRAINT `output_task_id` FOREIGN KEY (`task_id`) REFERENCES `tasks` (`id`) + ON UPDATE NO ACTION ON DELETE CASCADE, + CONSTRAINT `output_data_id` FOREIGN KEY (`data_id`) REFERENCES `data` (`id`) + ON UPDATE NO ACTION ON DELETE NO ACTION, + PRIMARY KEY (`task_id`, `position`) + ); + """, + """ + CREATE TABLE IF NOT EXISTS `task_inputs` ( + `task_id` BINARY(16) NOT NULL, + `position` INT UNSIGNED NOT NULL, + `type` VARCHAR(999) NOT NULL, + `output_task_id` BINARY(16), + `output_task_position` INT UNSIGNED, + `value` VARBINARY(999), + `data_id` BINARY(16), + CONSTRAINT `input_task_id` FOREIGN KEY (`task_id`) REFERENCES `tasks` (`id`) + ON UPDATE NO ACTION ON DELETE CASCADE, + CONSTRAINT `input_task_output_match` FOREIGN KEY (`output_task_id`, `output_task_position`) + REFERENCES task_outputs (`task_id`, `position`) ON UPDATE NO ACTION ON DELETE SET NULL, + CONSTRAINT `input_data_id` FOREIGN KEY (`data_id`) REFERENCES `data` (`id`) + ON UPDATE NO ACTION ON DELETE NO ACTION, + PRIMARY KEY (`task_id`, `position`) + ); + """, + """ + CREATE TABLE IF NOT EXISTS `task_dependencies` ( + `parent` BINARY(16) NOT NULL, + `child` BINARY(16) NOT NULL, + KEY (`parent`) USING BTREE, + KEY (`child`) USING BTREE, + CONSTRAINT `task_dep_parent` FOREIGN KEY (`parent`) REFERENCES `tasks` (`id`) + ON UPDATE NO ACTION ON DELETE CASCADE, + CONSTRAINT `task_dep_child` FOREIGN KEY (`child`) REFERENCES `tasks` (`id`) + ON UPDATE NO ACTION ON DELETE CASCADE + ); + """, + """ + CREATE TABLE IF NOT EXISTS `task_instances` ( + `id` BINARY(16) NOT NULL, + `task_id` BINARY(16) NOT NULL, + `start_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT `instance_task_id` FOREIGN KEY (`task_id`) REFERENCES `tasks` (`id`) + ON UPDATE NO ACTION ON DELETE CASCADE, + PRIMARY KEY (`id`) + ); + """, + """ + CREATE TABLE IF NOT EXISTS `scheduler_leases` ( + `scheduler_id` BINARY(16) NOT NULL, + `task_id` BINARY(16) NOT NULL, + `lease_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT `lease_scheduler_id` FOREIGN KEY (`scheduler_id`) REFERENCES `schedulers` (`id`) + ON UPDATE NO ACTION ON DELETE CASCADE, + CONSTRAINT `lease_task_id` FOREIGN KEY (`task_id`) REFERENCES `tasks` (`id`) + ON UPDATE NO ACTION ON DELETE CASCADE, + INDEX (`scheduler_id`), + PRIMARY KEY (`scheduler_id`, `task_id`) + ); + """, + """ + CREATE TABLE IF NOT EXISTS `data_locality` ( + `id` BINARY(16) NOT NULL, + `address` VARCHAR(40) NOT NULL, + KEY (`id`) USING BTREE, + CONSTRAINT `locality_data_id` FOREIGN KEY (`id`) REFERENCES `data` (`id`) + ON UPDATE NO ACTION ON DELETE CASCADE + ); + """, + """ + CREATE TABLE IF NOT EXISTS `data_ref_driver` ( + `id` BINARY(16) NOT NULL, + `driver_id` BINARY(16) NOT NULL, + KEY (`id`) USING BTREE, + KEY (`driver_id`) USING BTREE, + CONSTRAINT `data_driver_ref_id` FOREIGN KEY (`id`) REFERENCES `data` (`id`) + ON UPDATE NO ACTION ON DELETE CASCADE, + CONSTRAINT `data_ref_driver_id` FOREIGN KEY (`driver_id`) REFERENCES `drivers` (`id`) + ON UPDATE NO ACTION ON DELETE CASCADE + ); + """, + """ + CREATE TABLE IF NOT EXISTS `data_ref_task` ( + `id` BINARY(16) NOT NULL, + `task_id` BINARY(16) NOT NULL, + KEY (`id`) USING BTREE, + KEY (`task_id`) USING BTREE, + CONSTRAINT `data_task_ref_id` FOREIGN KEY (`id`) REFERENCES `data` (`id`) + ON UPDATE NO ACTION ON DELETE CASCADE, + CONSTRAINT `data_ref_task_id` FOREIGN KEY (`task_id`) REFERENCES `tasks` (`id`) + ON UPDATE NO ACTION ON DELETE CASCADE + ); + """, + """ + CREATE TABLE IF NOT EXISTS `client_kv_data` ( + `kv_key` VARCHAR(64) NOT NULL, + `value` VARBINARY(999) NOT NULL, + `client_id` BINARY(16) NOT NULL, + PRIMARY KEY (`client_id`, `kv_key`) + ); + """, + """ + CREATE TABLE IF NOT EXISTS `task_kv_data` ( + `kv_key` VARCHAR(64) NOT NULL, + `value` VARBINARY(999) NOT NULL, + `task_id` BINARY(16) NOT NULL, + PRIMARY KEY (`task_id`, `kv_key`), + CONSTRAINT `kv_data_task_id` FOREIGN KEY (`task_id`) REFERENCES `tasks` (`id`) + ON UPDATE NO ACTION ON DELETE CASCADE + ); + """, +] + + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", +) +logger = logging.getLogger(__name__) + + +def main() -> int: + """Main.""" + parser = argparse.ArgumentParser(description="Initialize the database tables for Spider.") + parser.add_argument( + "--port", + type=int, + default=3306, + help="The port MariaDB is hosting on (default: %(default)d)", + ) + parser.add_argument( + "--username", + type=str, + default="spider-user", + help="The username of the started MariaDB (default: %(default)s)", + ) + parser.add_argument( + "--password", + type=str, + default="spider-password", + help="The password of the started MariaDB (default: %(default)s)", + ) + parser.add_argument( + "--database", + type=str, + default="spider-db", + help="The database name of the started MariaDB (default: %(default)s)", + ) + args = parser.parse_args() + + with ( + mariadb.connect( + host="127.0.0.1", + port=args.port, + user=args.username, + password=args.password, + database=args.database, + ) as conn, + conn.cursor() as cursor, + ): + for table_creator in _TABLE_CREATORS: + cursor.execute(table_creator) + conn.commit() + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/scripts/storage/init_db.sql b/tools/scripts/storage/init_db.sql deleted file mode 100644 index b0a69aad..00000000 --- a/tools/scripts/storage/init_db.sql +++ /dev/null @@ -1,160 +0,0 @@ -CREATE TABLE IF NOT EXISTS `drivers` -( - `id` BINARY(16) NOT NULL, - `heartbeat` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`) -); -CREATE TABLE IF NOT EXISTS `schedulers` -( - `id` BINARY(16) NOT NULL, - `address` VARCHAR(40) NOT NULL, - `port` INT UNSIGNED NOT NULL, - CONSTRAINT `scheduler_driver_id` FOREIGN KEY (`id`) REFERENCES `drivers` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, - PRIMARY KEY (`id`) -); -CREATE TABLE IF NOT EXISTS jobs -( - `id` BINARY(16) NOT NULL, - `client_id` BINARY(16) NOT NULL, - `creation_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - `state` ENUM ('running', 'success', 'fail', 'cancel') NOT NULL DEFAULT 'running', - KEY (`client_id`) USING BTREE, - INDEX idx_jobs_creation_time (`creation_time`), - INDEX idx_jobs_state (`state`), - PRIMARY KEY (`id`) -); -CREATE TABLE IF NOT EXISTS tasks -( - `id` BINARY(16) NOT NULL, - `job_id` BINARY(16) NOT NULL, - `func_name` VARCHAR(64) NOT NULL, - `language` ENUM('cpp', 'python') NOT NULL, - `state` ENUM ('pending', 'ready', 'running', 'success', 'cancel', 'fail') NOT NULL, - `timeout` FLOAT, - `max_retry` INT UNSIGNED DEFAULT 0, - `retry` INT UNSIGNED DEFAULT 0, - `instance_id` BINARY(16), - CONSTRAINT `task_job_id` FOREIGN KEY (`job_id`) REFERENCES `jobs` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, - PRIMARY KEY (`id`) -); -CREATE TABLE IF NOT EXISTS input_tasks -( - `job_id` BINARY(16) NOT NULL, - `task_id` BINARY(16) NOT NULL, - `position` INT UNSIGNED NOT NULL, - CONSTRAINT `input_task_job_id` FOREIGN KEY (`job_id`) REFERENCES `jobs` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, - CONSTRAINT `input_task_task_id` FOREIGN KEY (`task_id`) REFERENCES `tasks` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, - INDEX (`job_id`, `position`), - PRIMARY KEY (`task_id`) -); -CREATE TABLE IF NOT EXISTS output_tasks -( - `job_id` BINARY(16) NOT NULL, - `task_id` BINARY(16) NOT NULL, - `position` INT UNSIGNED NOT NULL, - CONSTRAINT `output_task_job_id` FOREIGN KEY (`job_id`) REFERENCES `jobs` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, - CONSTRAINT `output_task_task_id` FOREIGN KEY (`task_id`) REFERENCES `tasks` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, - INDEX (`job_id`, `position`), - PRIMARY KEY (`task_id`) -); -CREATE TABLE IF NOT EXISTS `data` -( - `id` BINARY(16) NOT NULL, - `value` VARBINARY(999) NOT NULL, - `hard_locality` BOOL DEFAULT FALSE, - `persisted` BOOL DEFAULT FALSE, - PRIMARY KEY (`id`) -); -CREATE TABLE IF NOT EXISTS `task_outputs` -( - `task_id` BINARY(16) NOT NULL, - `position` INT UNSIGNED NOT NULL, - `type` VARCHAR(999) NOT NULL, - `value` VARBINARY(999), - `data_id` BINARY(16), - CONSTRAINT `output_task_id` FOREIGN KEY (`task_id`) REFERENCES `tasks` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, - CONSTRAINT `output_data_id` FOREIGN KEY (`data_id`) REFERENCES `data` (`id`) ON UPDATE NO ACTION ON DELETE NO ACTION, - PRIMARY KEY (`task_id`, `position`) -); -CREATE TABLE IF NOT EXISTS `task_inputs` -( - `task_id` BINARY(16) NOT NULL, - `position` INT UNSIGNED NOT NULL, - `type` VARCHAR(999) NOT NULL, - `output_task_id` BINARY(16), - `output_task_position` INT UNSIGNED, - `value` VARBINARY(999), -- Use VARBINARY for all types of values - `data_id` BINARY(16), - CONSTRAINT `input_task_id` FOREIGN KEY (`task_id`) REFERENCES `tasks` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, - CONSTRAINT `input_task_output_match` FOREIGN KEY (`output_task_id`, `output_task_position`) REFERENCES task_outputs (`task_id`, `position`) ON UPDATE NO ACTION ON DELETE SET NULL, - CONSTRAINT `input_data_id` FOREIGN KEY (`data_id`) REFERENCES `data` (`id`) ON UPDATE NO ACTION ON DELETE NO ACTION, - PRIMARY KEY (`task_id`, `position`) -); - -CREATE TABLE IF NOT EXISTS `task_dependencies` -( - `parent` BINARY(16) NOT NULL, - `child` BINARY(16) NOT NULL, - KEY (`parent`) USING BTREE, - KEY (`child`) USING BTREE, - CONSTRAINT `task_dep_parent` FOREIGN KEY (`parent`) REFERENCES `tasks` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, - CONSTRAINT `task_dep_child` FOREIGN KEY (`child`) REFERENCES `tasks` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE -); -CREATE TABLE IF NOT EXISTS `task_instances` -( - `id` BINARY(16) NOT NULL, - `task_id` BINARY(16) NOT NULL, - `start_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - CONSTRAINT `instance_task_id` FOREIGN KEY (`task_id`) REFERENCES `tasks` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, - PRIMARY KEY (`id`) -); -CREATE TABLE IF NOT EXISTS `scheduler_leases` -( - `scheduler_id` BINARY(16) NOT NULL, - `task_id` BINARY(16) NOT NULL, - `lease_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - CONSTRAINT `lease_scheduler_id` FOREIGN KEY (`scheduler_id`) REFERENCES `schedulers` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, - CONSTRAINT `lease_task_id` FOREIGN KEY (`task_id`) REFERENCES `tasks` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, - INDEX (`scheduler_id`), - PRIMARY KEY (`scheduler_id`, `task_id`) -); -CREATE TABLE IF NOT EXISTS `data_locality` -( - `id` BINARY(16) NOT NULL, - `address` VARCHAR(40) NOT NULL, - KEY (`id`) USING BTREE, - CONSTRAINT `locality_data_id` FOREIGN KEY (`id`) REFERENCES `data` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE -); -CREATE TABLE IF NOT EXISTS `data_ref_driver` -( - `id` BINARY(16) NOT NULL, - `driver_id` BINARY(16) NOT NULL, - KEY (`id`) USING BTREE, - KEY (`driver_id`) USING BTREE, - CONSTRAINT `data_driver_ref_id` FOREIGN KEY (`id`) REFERENCES `data` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, - CONSTRAINT `data_ref_driver_id` FOREIGN KEY (`driver_id`) REFERENCES `drivers` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE -); -CREATE TABLE IF NOT EXISTS `data_ref_task` -( - `id` BINARY(16) NOT NULL, - `task_id` BINARY(16) NOT NULL, - KEY (`id`) USING BTREE, - KEY (`task_id`) USING BTREE, - CONSTRAINT `data_task_ref_id` FOREIGN KEY (`id`) REFERENCES `data` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, - CONSTRAINT `data_ref_task_id` FOREIGN KEY (`task_id`) REFERENCES `tasks` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE -); -CREATE TABLE IF NOT EXISTS `client_kv_data` -( - `kv_key` VARCHAR(64) NOT NULL, - `value` VARBINARY(999) NOT NULL, - `client_id` BINARY(16) NOT NULL, - PRIMARY KEY (`client_id`, `kv_key`) -); -CREATE TABLE IF NOT EXISTS `task_kv_data` -( - `kv_key` VARCHAR(64) NOT NULL, - `value` VARBINARY(999) NOT NULL, - `task_id` BINARY(16) NOT NULL, - PRIMARY KEY (`task_id`, `kv_key`), - CONSTRAINT `kv_data_task_id` FOREIGN KEY (`task_id`) REFERENCES `tasks` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE -); \ No newline at end of file