Skip to content

Commit f18e073

Browse files
maxim-lixakovmaxim-lixakov
authored
[DOP-19901] - add integration MSSQL tests & CI (#145)
1 parent 950fb35 commit f18e073

File tree

17 files changed

+680
-5
lines changed

17 files changed

+680
-5
lines changed

‎.env.docker‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,14 @@ TEST_CLICKHOUSE_USER=default
8484
TEST_CLICKHOUSE_PASSWORD=
8585
TEST_CLICKHOUSE_DB=default
8686

87+
TEST_MSSQL_HOST_FOR_CONFTEST=test-mssql
88+
TEST_MSSQL_PORT_FOR_CONFTEST=1433
89+
TEST_MSSQL_HOST_FOR_WORKER=test-mssql
90+
TEST_MSSQL_PORT_FOR_WORKER=1433
91+
TEST_MSSQL_USER=syncmaster
92+
TEST_MSSQL_PASSWORD=7ellowEl7akey
93+
TEST_MSSQL_DB=syncmaster
94+
8795
TEST_HIVE_CLUSTER=test-hive
8896
TEST_HIVE_USER=syncmaster
8997
TEST_HIVE_PASSWORD=changeme

‎.env.local‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,14 @@ export TEST_CLICKHOUSE_USER=default
7070
export TEST_CLICKHOUSE_PASSWORD=
7171
export TEST_CLICKHOUSE_DB=default
7272

73+
export TEST_MSSQL_HOST_FOR_CONFTEST=localhost
74+
export TEST_MSSQL_PORT_FOR_CONFTEST=1433
75+
export TEST_MSSQL_HOST_FOR_WORKER=test-mssql
76+
export TEST_MSSQL_PORT_FOR_WORKER=1433
77+
export TEST_MSSQL_USER=syncmaster
78+
export TEST_MSSQL_PASSWORD=7ellowEl7akey
79+
export TEST_MSSQL_DB=syncmaster
80+
7381
export TEST_HIVE_CLUSTER=test-hive
7482
export TEST_HIVE_USER=syncmaster
7583
export TEST_HIVE_PASSWORD=changeme

‎.github/workflows/mssql-tests.yml‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
name: MSSQL Tests
2+
on:
3+
workflow_call:
4+
5+
env:
6+
DEFAULT_PYTHON: '3.12'
7+
8+
jobs:
9+
tests:
10+
name: Run MSSQL tests
11+
runs-on: ubuntu-latest
12+
13+
steps:
14+
- name: Checkout code
15+
uses: actions/checkout@v4
16+
17+
- name: Set up QEMU
18+
uses: docker/setup-qemu-action@v3
19+
20+
- name: Set up Docker Buildx
21+
uses: docker/setup-buildx-action@v3
22+
23+
- name: Cache jars
24+
uses: actions/cache@v4
25+
with:
26+
path: ./cached_jars
27+
key: ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-test-mssql
28+
restore-keys: |
29+
${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-test-mssql
30+
${{ runner.os }}-python-
31+
32+
- name: Build Worker Image
33+
uses: docker/build-push-action@v6
34+
with:
35+
context: .
36+
tags: mtsrus/syncmaster-worker:${{ github.sha }}
37+
target: test
38+
file: docker/Dockerfile.worker
39+
load: true
40+
cache-from: mtsrus/syncmaster-worker:develop
41+
42+
- name: Docker compose up
43+
run: |
44+
docker compose -f docker-compose.test.yml --profile all down -v --remove-orphans
45+
docker compose -f docker-compose.test.yml --profile mssql up -d --wait --wait-timeout 200
46+
env:
47+
WORKER_IMAGE_TAG: ${{ github.sha }}
48+
49+
- name: Run MSSQL Tests
50+
run: |
51+
docker compose -f ./docker-compose.test.yml --profile mssql exec -T worker coverage run -m pytest -vvv -s -m "worker and mssql"
52+
53+
- name: Dump worker logs on failure
54+
if: failure()
55+
uses: jwalton/gh-docker-logs@v2
56+
with:
57+
images: mtsrus/syncmaster-worker
58+
dest: ./logs
59+
60+
# This is important, as coverage is exported after receiving SIGTERM
61+
- name: Shutdown
62+
if: always()
63+
run: |
64+
docker compose -f docker-compose.test.yml --profile all down -v --remove-orphans
65+
66+
- name: Upload worker logs
67+
uses: actions/upload-artifact@v4
68+
if: failure()
69+
with:
70+
name: worker-logs-mssql
71+
path: logs/*
72+
73+
- name: Upload coverage results
74+
uses: actions/upload-artifact@v4
75+
with:
76+
name: coverage-mssql
77+
path: reports/*
78+
# https://github.com/actions/upload-artifact/issues/602
79+
include-hidden-files: true

‎.github/workflows/tests.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,10 @@ jobs:
2424
name: Clickhouse tests
2525
uses: ./.github/workflows/clickhouse-tests.yml
2626

27+
mssql_tests:
28+
name: MSSQL tests
29+
uses: ./.github/workflows/mssql-tests.yml
30+
2731
hdfs_tests:
2832
name: HDFS tests
2933
uses: ./.github/workflows/hdfs-tests.yml
@@ -48,7 +52,7 @@ jobs:
4852
name: Tests done
4953
runs-on: ubuntu-latest
5054

51-
needs: [oracle_tests, clickhouse_tests, hive_tests, hdfs_tests, s3_tests, unit_tests]
55+
needs: [oracle_tests, clickhouse_tests, mssql_tests, hive_tests, hdfs_tests, s3_tests, unit_tests]
5256
steps:
5357
- name: Checkout code
5458
uses: actions/checkout@v4

‎Makefile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,10 @@ test-integration-clickhouse: test-db ##@Test Run integration tests for Clickh
9393
docker compose -f docker-compose.test.yml --profile clickhouse up -d --wait $(DOCKER_COMPOSE_ARGS)
9494
${POETRY} run pytest ./tests/test_integration -m clickhouse $(PYTEST_ARGS)
9595

96+
test-integration-mssql: test-db ##@Test Run integration tests for MSSQL
97+
docker compose -f docker-compose.test.yml --profile mssql up -d --wait $(DOCKER_COMPOSE_ARGS)
98+
${POETRY} run pytest ./tests/test_integration -m mssql $(PYTEST_ARGS)
99+
96100
test-integration-oracle: test-db ##@Test Run integration tests for Oracle
97101
docker compose -f docker-compose.test.yml --profile oracle up -d --wait $(DOCKER_COMPOSE_ARGS)
98102
${POETRY} run pytest ./tests/test_integration -m oracle $(PYTEST_ARGS)

‎docker-compose.test.yml‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ services:
9595
condition: service_healthy
9696
rabbitmq:
9797
condition: service_healthy
98-
profiles: [worker, scheduler, s3, oracle, hdfs, hive, all, clickhouse]
98+
profiles: [worker, scheduler, s3, oracle, hdfs, hive, clickhouse, mssql, all]
9999

100100
test-postgres:
101101
image: postgres
@@ -109,7 +109,7 @@ services:
109109
interval: 30s
110110
timeout: 5s
111111
retries: 3
112-
profiles: [s3, oracle, clickhouse, hdfs, hive, all]
112+
profiles: [s3, oracle, clickhouse, mssql, hdfs, hive, all]
113113

114114
test-s3:
115115
image: bitnami/minio:latest
@@ -148,6 +148,24 @@ services:
148148
- 9001:9000
149149
profiles: [clickhouse, all]
150150

151+
test-mssql:
152+
image: mcr.microsoft.com/mssql/server
153+
restart: unless-stopped
154+
environment:
155+
ACCEPT_EULA: Y
156+
MSSQL_PID: Developer
157+
MSSQL_SA_PASSWORD: 7ellowEl7akey
158+
MSSQL_DATABASE: syncmaster
159+
MSSQL_USER: syncmaster
160+
MSSQL_PASSWORD: 7ellowEl7akey
161+
ports:
162+
- 1433:1433
163+
volumes:
164+
- ./docker/mssql/:/usr/config/
165+
entrypoint: [/usr/config/entrypoint.sh]
166+
platform: linux/amd64
167+
profiles: [mssql, all]
168+
151169
metastore-hive:
152170
image: postgres
153171
restart: unless-stopped

‎docker/mssql/configure-db.sh‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
#!/bin/bash
2+
set -o pipefail
3+
4+
# Wait 60 seconds for SQL Server to start up by ensuring that
5+
# calling SQLCMD does not return an error code, which will ensure that sqlcmd is accessible
6+
# and that system and user databases return "0" which means all databases are in an "online" state
7+
# https://docs.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-databases-transact-sql?view=sql-server-2017
8+
9+
declare DBSTATUS
10+
declare ERRCODE
11+
TIMEOUT=60
12+
START=$(date +%s)
13+
echo "Configure DB script started at $(date)"
14+
15+
# fix for https://github.com/microsoft/mssql-docker/issues/892
16+
if [[ -d "/opt/mssql-tools18/bin" ]]; then
17+
SQLCMD="/opt/mssql-tools18/bin/sqlcmd -No"
18+
else
19+
SQLCMD=/opt/mssql-tools/bin/sqlcmd
20+
fi
21+
22+
while true; do
23+
DELTA=$(($(date +%s) - START))
24+
if [[ $DELTA -gt $TIMEOUT ]]; then
25+
echo "ERROR: SQL Server took more than ${TIMEOUT} seconds to START up or one or more databases are not in an ONLINE state"
26+
exit 1
27+
fi
28+
29+
DBSTATUS=$($SQLCMD -h -1 -t 1 -U sa -P ${MSSQL_SA_PASSWORD} -Q "SET NOCOUNT ON; Select SUM(state) from sys.databases" 2>/dev/null | sed -e 's/^[[:space:]]*//')
30+
ERRCODE=$?
31+
if [[ "$DBSTATUS" -eq "0" && "$ERRCODE" -eq "0" ]]; then
32+
echo "INFO: Database ready."
33+
break
34+
else
35+
echo "INFO: Waiting for database to be ready..."
36+
sleep 1
37+
fi
38+
done
39+
40+
# Run the setup script to create the DB and the schema in the DB
41+
echo "Running setup.sql";
42+
$SQLCMD -S localhost -U sa -P $MSSQL_SA_PASSWORD -d master -i /usr/config/setup.sql;
43+
echo "Success";

‎docker/mssql/entrypoint.sh‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/bin/bash
2+
3+
# Start the script to create the DB and user
4+
/usr/config/configure-db.sh &
5+
6+
# Start SQL Server
7+
/opt/mssql/bin/sqlservr

‎docker/mssql/setup.sql‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/*
2+
3+
Enter custom T-SQL here that would run after SQL Server has started up.
4+
5+
*/
6+
7+
CREATE DATABASE syncmaster;
8+
GO
9+
10+
USE syncmaster;
11+
GO
12+
13+
CREATE LOGIN syncmaster WITH PASSWORD = '7ellowEl7akey';
14+
GO
15+
16+
CREATE USER syncmaster FOR LOGIN syncmaster;
17+
GO
18+
19+
GRANT CONTROL ON DATABASE::syncmaster TO syncmaster;
20+
GO

‎syncmaster/dto/connections.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,17 @@ class ClickhouseConnectionDTO(ConnectionDTO):
3131
type: ClassVar[str] = "clickhouse"
3232

3333

34+
@dataclass
35+
class MSSQLConnectionDTO(ConnectionDTO):
36+
host: str
37+
port: int
38+
user: str
39+
password: str
40+
database_name: str
41+
additional_params: dict
42+
type: ClassVar[str] = "mssql"
43+
44+
3445
@dataclass
3546
class OracleConnectionDTO(ConnectionDTO):
3647
host: str

0 commit comments

Comments
 (0)