[DOP-19899] - add integration Clickhouse tests & CI

maxim-lixakov · maxim-lixakov · commit 84894a2e6ba0 · 2024-11-25T11:57:36.000+03:00
diff --git a/.env.docker b/.env.docker
@@ -76,6 +76,14 @@ TEST_ORACLE_USER=syncmaster
 TEST_ORACLE_PASSWORD=changeme
 TEST_ORACLE_SERVICE_NAME=XEPDB1
 
+TEST_CLICKHOUSE_HOST_FOR_CONFTEST=test-clickhouse
+TEST_CLICKHOUSE_PORT_FOR_CONFTEST=8123
+TEST_CLICKHOUSE_HOST_FOR_WORKER=test-clickhouse
+TEST_CLICKHOUSE_PORT_FOR_WORKER=8123
+TEST_CLICKHOUSE_USER=default
+TEST_CLICKHOUSE_PASSWORD=
+TEST_CLICKHOUSE_DB=default
+
 TEST_HIVE_CLUSTER=test-hive
 TEST_HIVE_USER=syncmaster
 TEST_HIVE_PASSWORD=changeme
diff --git a/.env.local b/.env.local
@@ -62,6 +62,14 @@ export TEST_ORACLE_USER=syncmaster
 export TEST_ORACLE_PASSWORD=changeme
 export TEST_ORACLE_SERVICE_NAME=XEPDB1
 
+export TEST_CLICKHOUSE_HOST_FOR_CONFTEST=localhost
+export TEST_CLICKHOUSE_PORT_FOR_CONFTEST=8123
+export TEST_CLICKHOUSE_HOST_FOR_WORKER=test-clickhouse
+export TEST_CLICKHOUSE_PORT_FOR_WORKER=8123
+export TEST_CLICKHOUSE_USER=default
+export TEST_CLICKHOUSE_PASSWORD=
+export TEST_CLICKHOUSE_DB=default
+
 export TEST_HIVE_CLUSTER=test-hive
 export TEST_HIVE_USER=syncmaster
 export TEST_HIVE_PASSWORD=changeme
diff --git a/.github/workflows/clickhouse-tests.yml b/.github/workflows/clickhouse-tests.yml
@@ -0,0 +1,79 @@
+name: Clickhouse Tests
+on:
+  workflow_call:
+
+env:
+  DEFAULT_PYTHON: '3.12'
+
+jobs:
+  tests:
+    name: Run Clickhouse tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Cache jars
+        uses: actions/cache@v4
+        with:
+          path: ./cached_jars
+          key: ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-test-clickhouse
+          restore-keys: |
+            ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-test-clickhouse
+            ${{ runner.os }}-python-
+
+      - name: Build Worker Image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          tags: mtsrus/syncmaster-worker:${{ github.sha }}
+          target: test
+          file: docker/Dockerfile.worker
+          load: true
+          cache-from: mtsrus/syncmaster-worker:develop
+
+      - name: Docker compose up
+        run: |
+          docker compose -f docker-compose.test.yml --profile all down -v --remove-orphans
+          docker compose -f docker-compose.test.yml --profile clickhouse up -d --wait --wait-timeout 200
+        env:
+          WORKER_IMAGE_TAG: ${{ github.sha }}
+
+      - name: Run Clickhouse Tests
+        run: |
+          docker compose -f ./docker-compose.test.yml --profile clickhouse exec -T worker coverage run -m pytest -vvv -s -m "worker and clickhouse"
+
+      - name: Dump worker logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          images: mtsrus/syncmaster-worker
+          dest: ./logs
+
+      # This is important, as coverage is exported after receiving SIGTERM
+      - name: Shutdown
+        if: always()
+        run: |
+          docker compose -f docker-compose.test.yml --profile all down -v --remove-orphans
+
+      - name: Upload worker logs
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: worker-logs-clickhouse
+          path: logs/*
+
+      - name: Upload coverage results
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-clickhouse
+          path: reports/*
+          # https://github.com/actions/upload-artifact/issues/602
+          include-hidden-files: true
diff --git a/.github/workflows/hdfs-tests.yml b/.github/workflows/hdfs-tests.yml
@@ -46,7 +46,6 @@ jobs:
         env:
           WORKER_IMAGE_TAG: ${{ github.sha }}
 
-      # This is important, as coverage is exported after receiving SIGTERM
       - name: Run HDFS Tests
         run: |
           docker compose -f ./docker-compose.test.yml --profile hdfs exec -T worker coverage run -m pytest -vvv -s -m "worker and hdfs"
@@ -58,6 +57,7 @@ jobs:
           images: mtsrus/syncmaster-worker
           dest: ./logs
 
+      # This is important, as coverage is exported after receiving SIGTERM
       - name: Shutdown
         if: always()
         run: |
diff --git a/.github/workflows/hive-tests.yml b/.github/workflows/hive-tests.yml
@@ -46,7 +46,6 @@ jobs:
         env:
           WORKER_IMAGE_TAG: ${{ github.sha }}
 
-      # This is important, as coverage is exported after receiving SIGTERM
       - name: Run Hive Tests
         run: |
           docker compose -f ./docker-compose.test.yml --profile hive exec -T worker coverage run -m pytest -vvv -s -m "worker and hive"
@@ -58,6 +57,7 @@ jobs:
           images: mtsrus/syncmaster-worker
           dest: ./logs
 
+      # This is important, as coverage is exported after receiving SIGTERM
       - name: Shutdown
         if: always()
         run: |
diff --git a/.github/workflows/oracle-tests.yml b/.github/workflows/oracle-tests.yml
@@ -46,7 +46,6 @@ jobs:
         env:
           WORKER_IMAGE_TAG: ${{ github.sha }}
 
-      # This is important, as coverage is exported after receiving SIGTERM
       - name: Run Oracle Tests
         run: |
           docker compose -f ./docker-compose.test.yml --profile oracle exec -T worker coverage run -m pytest -vvv -s -m "worker and oracle"
@@ -58,6 +57,7 @@ jobs:
           images: mtsrus/syncmaster-worker
           dest: ./logs
 
+      # This is important, as coverage is exported after receiving SIGTERM
       - name: Shutdown
         if: always()
         run: |
diff --git a/.github/workflows/scheduler-tests.yml b/.github/workflows/scheduler-tests.yml
@@ -46,7 +46,6 @@ jobs:
         env:
           WORKER_IMAGE_TAG: ${{ github.sha }}
 
-      # This is important, as coverage is exported after receiving SIGTERM
       - name: Run Scheduler Tests
         run: |
           docker compose -f ./docker-compose.test.yml --profile worker exec -T worker coverage run -m pytest -vvv -s -m "worker and scheduler_integration"
@@ -58,6 +57,7 @@ jobs:
           images: mtsrus/syncmaster-worker
           dest: ./logs
 
+      # This is important, as coverage is exported after receiving SIGTERM
       - name: Shutdown
         if: always()
         run: |
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -20,6 +20,10 @@ jobs:
     name: Oracle tests
     uses: ./.github/workflows/oracle-tests.yml
 
+  clickhouse_tests:
+    name: Clickhouse tests
+    uses: ./.github/workflows/clickhouse-tests.yml
+
   hdfs_tests:
     name: HDFS tests
     uses: ./.github/workflows/hdfs-tests.yml
@@ -44,7 +48,7 @@ jobs:
     name: Tests done
     runs-on: ubuntu-latest
 
-    needs: [oracle_tests, hive_tests, hdfs_tests, s3_tests, unit_tests]
+    needs: [oracle_tests, clickhouse_tests, hive_tests, hdfs_tests, s3_tests, unit_tests]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
diff --git a/Makefile b/Makefile
@@ -65,10 +65,10 @@ broker-start: ##Broker Start broker
 
 
 
-test: test-db test-broker ##@Test Run tests
+test: test-db test-broker ##@Test           Run tests
 	${POETRY} run pytest $(PYTEST_ARGS)
 
-test-db: test-db-start db-upgrade ##@TestDB Prepare database (in docker)
+test-db: test-db-start db-upgrade ##@TestDBPrepare database (in docker)
 
 test-db-start: ##@TestDB Start database
 	docker compose -f docker-compose.test.yml up -d --wait db $(DOCKER_COMPOSE_ARGS)
@@ -78,33 +78,37 @@ test-broker: test-broker-start ##@TestBroker Prepare broker (in docker)
 test-broker-start: ##@TestBroker Start broker
 	docker compose -f docker-compose.test.yml up -d --wait rabbitmq $(DOCKER_COMPOSE_ARGS)
 
-test-unit: test-db ##@Test Run unit tests
+test-unit: test-db ##@Test           Run unit tests
 	${POETRY} run pytest ./tests/test_unit ./tests/test_database $(PYTEST_ARGS)
 
-test-integration-hdfs: test-db ##@Test Run integration tests for HDFS
+test-integration-hdfs: test-db ##@Test          Run integration tests for HDFS
 	docker compose -f docker-compose.test.yml --profile hdfs up -d --wait $(DOCKER_COMPOSE_ARGS)
 	${POETRY} run pytest ./tests/test_integration -m hdfs $(PYTEST_ARGS)
 
-test-integration-hive: test-db ##@Test Run integration tests for Hive
+test-integration-hive: test-db ##@Test          Run integration tests for Hive
 	docker compose -f docker-compose.test.yml --profile hive up -d --wait $(DOCKER_COMPOSE_ARGS)
 	${POETRY} run pytest ./tests/test_integration -m hive $(PYTEST_ARGS)
 
-test-integration-oracle: test-db ##@Test Run integration tests for Oracle
+test-integration-clickhouse: test-db ##@Test    Run integration tests for Clickhouse
+	docker compose -f docker-compose.test.yml --profile clickhouse up -d --wait $(DOCKER_COMPOSE_ARGS)
+	${POETRY} run pytest ./tests/test_integration -m clickhouse $(PYTEST_ARGS)
+
+test-integration-oracle: test-db ##@Test        Run integration tests for Oracle
 	docker compose -f docker-compose.test.yml --profile oracle up -d --wait $(DOCKER_COMPOSE_ARGS)
 	${POETRY} run pytest ./tests/test_integration -m oracle $(PYTEST_ARGS)
 
-test-integration-s3: test-db ##@Test Run integration tests for S3
+test-integration-s3: test-db ##@Test           Run integration tests for S3
 	docker compose -f docker-compose.test.yml --profile s3 up -d --wait $(DOCKER_COMPOSE_ARGS)
 	${POETRY} run pytest ./tests/test_integration -m s3 $(PYTEST_ARGS)
 
-test-integration: test-db ##@Test Run all integration tests
+test-integration: test-db ##@Test           Run all integration tests
 	docker compose -f docker-compose.test.yml --profile all up -d --wait $(DOCKER_COMPOSE_ARGS)
 	${POETRY} run pytest ./tests/test_integration $(PYTEST_ARGS)
 
-test-check-fixtures: ##@Test Check declared fixtures
+test-check-fixtures: ##@Test           Check declared fixtures
 	${POETRY} run pytest --dead-fixtures $(PYTEST_ARGS)
 
-test-cleanup: ##@Test Cleanup tests dependencies
+test-cleanup: ##@Test           Cleanup tests dependencies
 	docker compose -f docker-compose.test.yml --profile all down $(ARGS)
 
 
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
@@ -95,7 +95,7 @@ services:
         condition: service_healthy
       rabbitmq:
         condition: service_healthy
-    profiles: [worker, scheduler, s3, oracle, hdfs, hive, all]
+    profiles: [worker, scheduler, s3, oracle, hdfs, hive, all, clickhouse]
 
   test-postgres:
     image: postgres
@@ -109,7 +109,7 @@ services:
       interval: 30s
       timeout: 5s
       retries: 3
-    profiles: [s3, oracle, hdfs, hive, all]
+    profiles: [s3, oracle, clickhouse, hdfs, hive, all]
 
   test-s3:
     image: bitnami/minio:latest
@@ -140,6 +140,14 @@ services:
       APP_USER_PASSWORD: changeme
     profiles: [oracle, all]
 
+  test-clickhouse:
+    image: clickhouse/clickhouse-server
+    restart: unless-stopped
+    ports:
+      - 8123:8123
+      - 9001:9000
+    profiles: [clickhouse, all]
+
   metastore-hive:
     image: postgres
     restart: unless-stopped
diff --git a/syncmaster/dto/connections.py b/syncmaster/dto/connections.py
@@ -20,6 +20,17 @@ class PostgresConnectionDTO(ConnectionDTO):
     type: ClassVar[str] = "postgres"
 
 
+@dataclass
+class ClickhouseConnectionDTO(ConnectionDTO):
+    host: str
+    port: int
+    user: str
+    password: str
+    database_name: str
+    additional_params: dict
+    type: ClassVar[str] = "clickhouse"
+
+
 @dataclass
 class OracleConnectionDTO(ConnectionDTO):
     host: str
diff --git a/syncmaster/dto/transfers.py b/syncmaster/dto/transfers.py
@@ -51,6 +51,11 @@ class OracleTransferDTO(DBTransferDTO):
     type: ClassVar[str] = "oracle"
 
 
+@dataclass
+class ClickhouseTransferDTO(DBTransferDTO):
+    type: ClassVar[str] = "clickhouse"
+
+
 @dataclass
 class HiveTransferDTO(DBTransferDTO):
     type: ClassVar[str] = "hive"
diff --git a/syncmaster/worker/controller.py b/syncmaster/worker/controller.py
@@ -5,13 +5,15 @@
 
 from syncmaster.db.models import Connection, Run
 from syncmaster.dto.connections import (
+    ClickhouseConnectionDTO,
     HDFSConnectionDTO,
     HiveConnectionDTO,
     OracleConnectionDTO,
     PostgresConnectionDTO,
     S3ConnectionDTO,
 )
 from syncmaster.dto.transfers import (
+    ClickhouseTransferDTO,
     HDFSTransferDTO,
     HiveTransferDTO,
     OracleTransferDTO,
@@ -20,6 +22,7 @@
 )
 from syncmaster.exceptions.connection import ConnectionTypeNotRecognizedError
 from syncmaster.worker.handlers.base import Handler
+from syncmaster.worker.handlers.db.clickhouse import ClickhouseHandler
 from syncmaster.worker.handlers.db.hive import HiveHandler
 from syncmaster.worker.handlers.db.oracle import OracleHandler
 from syncmaster.worker.handlers.db.postgres import PostgresHandler
@@ -41,6 +44,11 @@
         OracleConnectionDTO,
         OracleTransferDTO,
     ),
+    "clickhouse": (
+        ClickhouseHandler,
+        ClickhouseConnectionDTO,
+        ClickhouseTransferDTO,
+    ),
     "postgres": (
         PostgresHandler,
         PostgresConnectionDTO,
diff --git a/syncmaster/worker/handlers/db/clickhouse.py b/syncmaster/worker/handlers/db/clickhouse.py
diff --git a/syncmaster/worker/spark.py b/syncmaster/worker/spark.py
diff --git a/tests/settings.py b/tests/settings.py
diff --git a/tests/test_integration/test_run_transfer/conftest.py b/tests/test_integration/test_run_transfer/conftest.py
diff --git a/tests/test_integration/test_run_transfer/test_clickhouse.py b/tests/test_integration/test_run_transfer/test_clickhouse.py
