Skip to content

Commit e7c10e9

Browse files
author
maxim-lixakov
committed
[DOP-19899] - add integration Clickhouse tests & CI
1 parent 32726c9 commit e7c10e9

File tree

15 files changed

+593
-3
lines changed

15 files changed

+593
-3
lines changed

‎.env.docker‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,14 @@ TEST_ORACLE_USER=syncmaster
7676
TEST_ORACLE_PASSWORD=changeme
7777
TEST_ORACLE_SERVICE_NAME=XEPDB1
7878

79+
TEST_CLICKHOUSE_HOST_FOR_CONFTEST=test-clickhouse
80+
TEST_CLICKHOUSE_PORT_FOR_CONFTEST=8123
81+
TEST_CLICKHOUSE_HOST_FOR_WORKER=test-clickhouse
82+
TEST_CLICKHOUSE_PORT_FOR_WORKER=8123
83+
TEST_CLICKHOUSE_USER=default
84+
TEST_CLICKHOUSE_PASSWORD=
85+
TEST_CLICKHOUSE_DB=default
86+
7987
TEST_HIVE_CLUSTER=test-hive
8088
TEST_HIVE_USER=syncmaster
8189
TEST_HIVE_PASSWORD=changeme

‎.env.local‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,14 @@ export TEST_ORACLE_USER=syncmaster
6262
export TEST_ORACLE_PASSWORD=changeme
6363
export TEST_ORACLE_SERVICE_NAME=XEPDB1
6464

65+
export TEST_CLICKHOUSE_HOST_FOR_CONFTEST=localhost
66+
export TEST_CLICKHOUSE_PORT_FOR_CONFTEST=8123
67+
export TEST_CLICKHOUSE_HOST_FOR_WORKER=test-clickhouse
68+
export TEST_CLICKHOUSE_PORT_FOR_WORKER=8123
69+
export TEST_CLICKHOUSE_USER=default
70+
export TEST_CLICKHOUSE_PASSWORD=
71+
export TEST_CLICKHOUSE_DB=default
72+
6573
export TEST_HIVE_CLUSTER=test-hive
6674
export TEST_HIVE_USER=syncmaster
6775
export TEST_HIVE_PASSWORD=changeme

‎.github/workflows/clickhouse-tests.yml‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
name: Clickhouse Tests
2+
on:
3+
workflow_call:
4+
5+
env:
6+
DEFAULT_PYTHON: '3.12'
7+
8+
jobs:
9+
tests:
10+
name: Run Clickhouse tests
11+
runs-on: ubuntu-latest
12+
13+
steps:
14+
- name: Checkout code
15+
uses: actions/checkout@v4
16+
17+
- name: Set up QEMU
18+
uses: docker/setup-qemu-action@v3
19+
20+
- name: Set up Docker Buildx
21+
uses: docker/setup-buildx-action@v3
22+
23+
- name: Cache jars
24+
uses: actions/cache@v4
25+
with:
26+
path: ./cached_jars
27+
key: ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-test-clickhouse
28+
restore-keys: |
29+
${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-test-clickhouse
30+
${{ runner.os }}-python-
31+
32+
- name: Build Worker Image
33+
uses: docker/build-push-action@v6
34+
with:
35+
context: .
36+
tags: mtsrus/syncmaster-worker:${{ github.sha }}
37+
target: test
38+
file: docker/Dockerfile.worker
39+
load: true
40+
cache-from: mtsrus/syncmaster-worker:develop
41+
42+
- name: Docker compose up
43+
run: |
44+
docker compose -f docker-compose.test.yml --profile all down -v --remove-orphans
45+
docker compose -f docker-compose.test.yml --profile clickhouse up -d --wait --wait-timeout 200
46+
env:
47+
WORKER_IMAGE_TAG: ${{ github.sha }}
48+
49+
# This is important, as coverage is exported after receiving SIGTERM
50+
- name: Run Clickhouse Tests
51+
run: |
52+
docker compose -f ./docker-compose.test.yml --profile clickhouse exec -T worker coverage run -m pytest -vvv -s -m "worker and clickhouse"
53+
54+
- name: Dump worker logs on failure
55+
if: failure()
56+
uses: jwalton/gh-docker-logs@v2
57+
with:
58+
images: mtsrus/syncmaster-worker
59+
dest: ./logs
60+
61+
- name: Shutdown
62+
if: always()
63+
run: |
64+
docker compose -f docker-compose.test.yml --profile all down -v --remove-orphans
65+
66+
- name: Upload worker logs
67+
uses: actions/upload-artifact@v4
68+
if: failure()
69+
with:
70+
name: worker-logs-clickhouse
71+
path: logs/*
72+
73+
- name: Upload coverage results
74+
uses: actions/upload-artifact@v4
75+
with:
76+
name: coverage-clickhouse
77+
path: reports/*
78+
# https://github.com/actions/upload-artifact/issues/602
79+
include-hidden-files: true

‎.github/workflows/tests.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@ jobs:
2020
name: Oracle tests
2121
uses: ./.github/workflows/oracle-tests.yml
2222

23+
clickhouse_tests:
24+
name: Clickhouse tests
25+
uses: ./.github/workflows/clickhouse-tests.yml
26+
2327
hdfs_tests:
2428
name: HDFS tests
2529
uses: ./.github/workflows/hdfs-tests.yml
@@ -44,7 +48,7 @@ jobs:
4448
name: Tests done
4549
runs-on: ubuntu-latest
4650

47-
needs: [oracle_tests, hive_tests, hdfs_tests, s3_tests, unit_tests]
51+
needs: [oracle_tests, clickhouse_tests, hive_tests, hdfs_tests, s3_tests, unit_tests]
4852
steps:
4953
- name: Checkout code
5054
uses: actions/checkout@v4

‎Makefile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,10 @@ test-integration-hive: test-db ##@Test Run integration tests for Hive
8989
docker compose -f docker-compose.test.yml --profile hive up -d --wait $(DOCKER_COMPOSE_ARGS)
9090
${POETRY} run pytest ./tests/test_integration -m hive $(PYTEST_ARGS)
9191

92+
test-integration-clickhouse: test-db ##@Test Run integration tests for Clickhouse
93+
docker compose -f docker-compose.test.yml --profile clickhouse up -d --wait $(DOCKER_COMPOSE_ARGS)
94+
${POETRY} run pytest ./tests/test_integration -m clickhouse $(PYTEST_ARGS)
95+
9296
test-integration-oracle: test-db ##@Test Run integration tests for Oracle
9397
docker compose -f docker-compose.test.yml --profile oracle up -d --wait $(DOCKER_COMPOSE_ARGS)
9498
${POETRY} run pytest ./tests/test_integration -m oracle $(PYTEST_ARGS)

‎docker-compose.test.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,14 @@ services:
140140
APP_USER_PASSWORD: changeme
141141
profiles: [oracle, all]
142142

143+
test-clickhouse:
144+
image: clickhouse/clickhouse-server
145+
restart: unless-stopped
146+
ports:
147+
- 8123:8123
148+
- 9001:9000
149+
profiles: [clickhouse, all]
150+
143151
metastore-hive:
144152
image: postgres
145153
restart: unless-stopped

‎syncmaster/dto/connections.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,17 @@ class PostgresConnectionDTO(ConnectionDTO):
2020
type: ClassVar[str] = "postgres"
2121

2222

23+
@dataclass
24+
class ClickhouseConnectionDTO(ConnectionDTO):
25+
host: str
26+
port: int
27+
user: str
28+
password: str
29+
database_name: str
30+
additional_params: dict
31+
type: ClassVar[str] = "clickhouse"
32+
33+
2334
@dataclass
2435
class OracleConnectionDTO(ConnectionDTO):
2536
host: str

‎syncmaster/dto/transfers.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,11 @@ class OracleTransferDTO(DBTransferDTO):
5151
type: ClassVar[str] = "oracle"
5252

5353

54+
@dataclass
55+
class ClickhouseTransferDTO(DBTransferDTO):
56+
type: ClassVar[str] = "clickhouse"
57+
58+
5459
@dataclass
5560
class HiveTransferDTO(DBTransferDTO):
5661
type: ClassVar[str] = "hive"

‎syncmaster/worker/controller.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,15 @@
55

66
from syncmaster.db.models import Connection, Run
77
from syncmaster.dto.connections import (
8+
ClickhouseConnectionDTO,
89
HDFSConnectionDTO,
910
HiveConnectionDTO,
1011
OracleConnectionDTO,
1112
PostgresConnectionDTO,
1213
S3ConnectionDTO,
1314
)
1415
from syncmaster.dto.transfers import (
16+
ClickhouseTransferDTO,
1517
HDFSTransferDTO,
1618
HiveTransferDTO,
1719
OracleTransferDTO,
@@ -20,6 +22,7 @@
2022
)
2123
from syncmaster.exceptions.connection import ConnectionTypeNotRecognizedError
2224
from syncmaster.worker.handlers.base import Handler
25+
from syncmaster.worker.handlers.db.clickhouse import ClickhouseHandler
2326
from syncmaster.worker.handlers.db.hive import HiveHandler
2427
from syncmaster.worker.handlers.db.oracle import OracleHandler
2528
from syncmaster.worker.handlers.db.postgres import PostgresHandler
@@ -41,6 +44,11 @@
4144
OracleConnectionDTO,
4245
OracleTransferDTO,
4346
),
47+
"clickhouse": (
48+
ClickhouseHandler,
49+
ClickhouseConnectionDTO,
50+
ClickhouseTransferDTO,
51+
),
4452
"postgres": (
4553
PostgresHandler,
4654
PostgresConnectionDTO,

‎syncmaster/worker/handlers/db/base.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
from typing import TYPE_CHECKING
88

99
from onetl.base import BaseDBConnection
10+
from onetl.connection import Clickhouse
1011
from onetl.db import DBReader, DBWriter
1112

1213
from syncmaster.dto.transfers import DBTransferDTO
@@ -31,6 +32,12 @@ def write(self, df: DataFrame) -> None:
3132
writer = DBWriter(
3233
connection=self.connection,
3334
table=self.transfer_dto.table_name,
35+
# TODO: get options from transfer_dto
36+
options=(
37+
Clickhouse.WriteOptions(createTableOptions="ENGINE = TinyLog")
38+
if self.transfer_dto.type == "clickhouse"
39+
else None
40+
),
3441
)
3542
return writer.run(df=self.normalize_column_names(df))
3643

0 commit comments

Comments
 (0)