Skip to content

Commit a217fa5

Browse files
IlyasDevelopmentIlyasDevelopment
authored
[DOP-22337] Add logic for handling Samba transfers (#192)
1 parent cc69298 commit a217fa5

File tree

26 files changed

+689
-302
lines changed

26 files changed

+689
-302
lines changed

‎.env.docker‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,19 @@ TEST_FTPS_PORT_FOR_WORKER=21
129129
TEST_FTPS_USER=syncmaster
130130
TEST_FTPS_PASSWORD=test_only
131131

132+
TEST_SAMBA_HOST_FOR_CONFTEST=test-samba
133+
TEST_SAMBA_PORT_FOR_CONFTEST=445
134+
TEST_SAMBA_HOST_FOR_WORKER=test-samba
135+
TEST_SAMBA_PORT_FOR_WORKER=445
136+
TEST_SAMBA_DOMAIN=domain
137+
TEST_SAMBA_PROTOCOL=SMB
138+
TEST_SAMBA_UID=1000
139+
TEST_SAMBA_GID=1000
140+
TEST_SAMBA_SHARE=SmbShare
141+
TEST_SAMBA_USER=syncmaster
142+
TEST_SAMBA_PASSWORD=test_only
143+
TEST_SAMBA_AUTH_TYPE=NTLMv2
144+
132145
SPARK_CONF_DIR=/app/tests/spark/hive/conf/
133146
HADOOP_CONF_DIR=/app/tests/spark/hadoop/
134147
HIVE_CONF_DIR=/app/tests/spark/hive/conf/

‎.env.local‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,19 @@ export TEST_FTPS_PORT_FOR_WORKER=21
116116
export TEST_FTPS_USER=syncmaster
117117
export TEST_FTPS_PASSWORD=test_only
118118

119+
export TEST_SAMBA_HOST_FOR_CONFTEST=localhost
120+
export TEST_SAMBA_PORT_FOR_CONFTEST=1445
121+
export TEST_SAMBA_HOST_FOR_WORKER=test-samba
122+
export TEST_SAMBA_PORT_FOR_WORKER=445
123+
export TEST_SAMBA_DOMAIN=domain
124+
export TEST_SAMBA_PROTOCOL=SMB
125+
export TEST_SAMBA_UID=1000
126+
export TEST_SAMBA_GID=1000
127+
export TEST_SAMBA_SHARE=SmbShare
128+
export TEST_SAMBA_USER=syncmaster
129+
export TEST_SAMBA_PASSWORD=test_only
130+
export TEST_SAMBA_AUTH_TYPE=NTLMv2
131+
119132
export SPARK_CONF_DIR=./tests/spark/hive/conf/
120133
export HADOOP_CONF_DIR=./tests/spark/hadoop/
121134
export HIVE_CONF_DIR=./tests/spark/hive/conf/

‎.github/workflows/samba-tests.yml‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
name: Samba tests
2+
on:
3+
workflow_call:
4+
5+
env:
6+
DEFAULT_PYTHON: '3.12'
7+
8+
jobs:
9+
test:
10+
name: Run Samba tests
11+
runs-on: ubuntu-latest
12+
13+
steps:
14+
- name: Checkout code
15+
uses: actions/checkout@v4
16+
17+
- name: Set up QEMU
18+
uses: docker/setup-qemu-action@v3
19+
20+
- name: Set up Docker Buildx
21+
uses: docker/setup-buildx-action@v3
22+
23+
- name: Cache jars
24+
uses: actions/cache@v4
25+
with:
26+
path: ./cached_jars
27+
key: ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-test-samba
28+
restore-keys: |
29+
${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-test-samba
30+
${{ runner.os }}-python-
31+
32+
- name: Build Worker Image
33+
uses: docker/build-push-action@v6
34+
with:
35+
context: .
36+
tags: mtsrus/syncmaster-worker:${{ github.sha }}
37+
target: test
38+
file: docker/Dockerfile.worker
39+
load: true
40+
cache-from: mtsrus/syncmaster-worker:develop
41+
42+
- name: Make custom entrypoint script executable
43+
run: chmod +x ./docker/samba/custom_entrypoint.sh
44+
45+
- name: Docker compose up
46+
run: |
47+
docker compose -f docker-compose.test.yml --profile all down -v --remove-orphans
48+
docker compose -f docker-compose.test.yml --profile samba up -d --wait --wait-timeout 200
49+
env:
50+
WORKER_IMAGE_TAG: ${{ github.sha }}
51+
52+
- name: Run Samba Tests
53+
run: |
54+
docker compose -f ./docker-compose.test.yml --profile samba exec -T worker coverage run -m pytest -vvv -s -m "worker and samba"
55+
56+
- name: Dump worker logs on failure
57+
if: failure()
58+
uses: jwalton/gh-docker-logs@v2
59+
with:
60+
images: mtsrus/syncmaster-worker
61+
dest: ./logs
62+
63+
# This is important, as coverage is exported after receiving SIGTERM
64+
- name: Shutdown
65+
if: always()
66+
run: |
67+
docker compose -f docker-compose.test.yml --profile all down -v --remove-orphans
68+
69+
- name: Upload worker logs
70+
uses: actions/upload-artifact@v4
71+
if: failure()
72+
with:
73+
name: worker-logs-samba
74+
path: logs/*
75+
76+
- name: Upload coverage results
77+
uses: actions/upload-artifact@v4
78+
with:
79+
name: coverage-samba
80+
path: reports/*
81+
# https://github.com/actions/upload-artifact/issues/602
82+
include-hidden-files: true

‎.github/workflows/tests.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,10 @@ jobs:
5656
name: FTPS tests
5757
uses: ./.github/workflows/ftps-tests.yml
5858

59+
samba_tests:
60+
name: Samba tests
61+
uses: ./.github/workflows/samba-tests.yml
62+
5963
scheduler_tests:
6064
name: Scheduler tests
6165
uses: ./.github/workflows/scheduler-tests.yml
@@ -68,7 +72,7 @@ jobs:
6872
name: Tests done
6973
runs-on: ubuntu-latest
7074

71-
needs: [unit_tests, scheduler_tests, oracle_tests, clickhouse_tests, mssql_tests, mysql_tests, hive_tests, hdfs_tests, s3_tests, sftp_tests, ftp_tests, ftps_tests]
75+
needs: [unit_tests, scheduler_tests, oracle_tests, clickhouse_tests, mssql_tests, mysql_tests, hive_tests, hdfs_tests, s3_tests, sftp_tests, ftp_tests, ftps_tests, samba_tests]
7276
steps:
7377
- name: Checkout code
7478
uses: actions/checkout@v4

‎Makefile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,10 @@ test-integration-ftps: test-db ##@Test Run integration tests for FTPS
121121
docker compose -f docker-compose.test.yml --profile ftps up -d --wait $(DOCKER_COMPOSE_ARGS)
122122
${POETRY} run pytest ./tests/test_integration -m ftps $(PYTEST_ARGS)
123123

124+
test-integration-samba: test-db ##@Test Run integration tests for Samba
125+
docker compose -f docker-compose.test.yml --profile samba up -d --wait $(DOCKER_COMPOSE_ARGS)
126+
${POETRY} run pytest ./tests/test_integration -m samba $(PYTEST_ARGS)
127+
124128
test-integration: test-db ##@Test Run all integration tests
125129
docker compose -f docker-compose.test.yml --profile all up -d --wait $(DOCKER_COMPOSE_ARGS)
126130
${POETRY} run pytest ./tests/test_integration $(PYTEST_ARGS)

‎README.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ List of currently supported connections:
4343
* FTP
4444
* FTPS
4545
* SFTP
46+
* Samba
4647

4748
Current Data.SyncMaster implementation provides following components:
4849

‎docker-compose.test.yml‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ services:
125125
condition: service_completed_successfully
126126
rabbitmq:
127127
condition: service_healthy
128-
profiles: [worker, scheduler, s3, oracle, hdfs, hive, clickhouse, mysql, mssql, sftp, ftp, ftps, all]
128+
profiles: [worker, scheduler, s3, oracle, hdfs, hive, clickhouse, mysql, mssql, sftp, ftp, ftps, samba, all]
129129

130130
test-postgres:
131131
image: postgres
@@ -139,7 +139,7 @@ services:
139139
interval: 30s
140140
timeout: 5s
141141
retries: 3
142-
profiles: [s3, oracle, clickhouse, mysql, mssql, hdfs, hive, sftp, ftp, ftps, all]
142+
profiles: [s3, oracle, clickhouse, mysql, mssql, hdfs, hive, sftp, ftp, ftps, samba, all]
143143

144144
test-s3:
145145
image: bitnami/minio:latest
@@ -225,7 +225,7 @@ services:
225225
interval: 30s
226226
timeout: 5s
227227
retries: 3
228-
profiles: [hive, hdfs, s3, sftp, ftp, ftps, all]
228+
profiles: [hive, hdfs, s3, sftp, ftp, ftps, samba, all]
229229

230230
keycloak:
231231
image: quay.io/keycloak/keycloak:latest
@@ -264,7 +264,7 @@ services:
264264
HIVE_METASTORE_DB_USER: test_hive
265265
HIVE_METASTORE_DB_PASSWORD: test_hive
266266
# writing spark dataframe to s3, sftp, ftp, ftps xml file fails without running hive metastore server
267-
profiles: [hive, hdfs, s3, sftp, ftp, ftps, all]
267+
profiles: [hive, hdfs, s3, sftp, ftp, ftps, samba, all]
268268

269269
test-sftp:
270270
image: ${SFTP_IMAGE:-linuxserver/openssh-server}
@@ -316,6 +316,17 @@ services:
316316
- ./docker/ftp/on_post_init.sh:/sources/ftps/eventscripts/on_post_init.sh
317317
profiles: [ftps, all]
318318

319+
test-samba:
320+
image: ${SAMBA_IMAGE:-elswork/samba}
321+
restart: unless-stopped
322+
ports:
323+
- 139:139
324+
- 1445:445
325+
volumes:
326+
- ./docker/samba/custom_entrypoint.sh:/custom_entrypoint.sh
327+
entrypoint: [/custom_entrypoint.sh]
328+
profiles: [samba, all]
329+
319330
volumes:
320331
postgres_test_data:
321332
rabbitmq_test_data:

‎docker/samba/custom_entrypoint.sh‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/usr/bin/env bash
2+
3+
# allow create files and directories
4+
mkdir -p /share/folder
5+
chmod 0777 /share/folder
6+
/entrypoint.sh -u "1000:1000:syncmaster:syncmaster:test_only" -s "SmbShare:/share/folder:rw:syncmaster"

‎docs/changelog/next_release/192.feature.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Add logic for handling Samba transfers

‎syncmaster/dto/connections.py‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
22
# SPDX-License-Identifier: Apache-2.0
33
from dataclasses import dataclass
4-
from typing import ClassVar
4+
from typing import ClassVar, Literal
55

66

77
@dataclass
@@ -119,3 +119,16 @@ class FTPSConnectionDTO(ConnectionDTO):
119119
user: str
120120
password: str
121121
type: ClassVar[str] = "ftps"
122+
123+
124+
@dataclass
125+
class SambaConnectionDTO(ConnectionDTO):
126+
host: str
127+
share: str
128+
protocol: Literal["SMB", "NetBIOS"]
129+
user: str
130+
password: str
131+
auth_type: Literal["NTLMv1", "NTLMv2"] = "NTLMv2"
132+
domain: str = ""
133+
port: int | None = None
134+
type: ClassVar[str] = "samba"

0 commit comments

Comments
 (0)