Skip to content

Commit cc69298

Browse files
IlyasDevelopmentIlyasDevelopment
authored
[DOP-22142] Add logic for handling FTP|FTPS transfers (#191)
1 parent 8adef5c commit cc69298

File tree

25 files changed

+1383
-12
lines changed

25 files changed

+1383
-12
lines changed

‎.env.docker‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,20 @@ TEST_SFTP_PORT_FOR_WORKER=2222
115115
TEST_SFTP_USER=syncmaster
116116
TEST_SFTP_PASSWORD=test_only
117117

118+
TEST_FTP_HOST_FOR_CONFTEST=test-ftp
119+
TEST_FTP_PORT_FOR_CONFTEST=21
120+
TEST_FTP_HOST_FOR_WORKER=test-ftp
121+
TEST_FTP_PORT_FOR_WORKER=21
122+
TEST_FTP_USER=syncmaster
123+
TEST_FTP_PASSWORD=test_only
124+
125+
TEST_FTPS_HOST_FOR_CONFTEST=test-ftps
126+
TEST_FTPS_PORT_FOR_CONFTEST=21
127+
TEST_FTPS_HOST_FOR_WORKER=test-ftps
128+
TEST_FTPS_PORT_FOR_WORKER=21
129+
TEST_FTPS_USER=syncmaster
130+
TEST_FTPS_PASSWORD=test_only
131+
118132
SPARK_CONF_DIR=/app/tests/spark/hive/conf/
119133
HADOOP_CONF_DIR=/app/tests/spark/hadoop/
120134
HIVE_CONF_DIR=/app/tests/spark/hive/conf/

‎.env.local‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,20 @@ export TEST_SFTP_PORT_FOR_WORKER=2222
102102
export TEST_SFTP_USER=syncmaster
103103
export TEST_SFTP_PASSWORD=test_only
104104

105+
export TEST_FTP_HOST_FOR_CONFTEST=localhost
106+
export TEST_FTP_PORT_FOR_CONFTEST=2121
107+
export TEST_FTP_HOST_FOR_WORKER=test-ftp
108+
export TEST_FTP_PORT_FOR_WORKER=21
109+
export TEST_FTP_USER=syncmaster
110+
export TEST_FTP_PASSWORD=test_only
111+
112+
export TEST_FTPS_HOST_FOR_CONFTEST=localhost
113+
export TEST_FTPS_PORT_FOR_CONFTEST=2122
114+
export TEST_FTPS_HOST_FOR_WORKER=test-ftps
115+
export TEST_FTPS_PORT_FOR_WORKER=21
116+
export TEST_FTPS_USER=syncmaster
117+
export TEST_FTPS_PASSWORD=test_only
118+
105119
export SPARK_CONF_DIR=./tests/spark/hive/conf/
106120
export HADOOP_CONF_DIR=./tests/spark/hadoop/
107121
export HIVE_CONF_DIR=./tests/spark/hive/conf/

‎.github/workflows/ftp-tests.yml‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
name: FTP tests
2+
on:
3+
workflow_call:
4+
5+
env:
6+
DEFAULT_PYTHON: '3.12'
7+
8+
jobs:
9+
test:
10+
name: Run FTP tests
11+
runs-on: ubuntu-latest
12+
13+
steps:
14+
- name: Checkout code
15+
uses: actions/checkout@v4
16+
17+
- name: Set up QEMU
18+
uses: docker/setup-qemu-action@v3
19+
20+
- name: Set up Docker Buildx
21+
uses: docker/setup-buildx-action@v3
22+
23+
- name: Cache jars
24+
uses: actions/cache@v4
25+
with:
26+
path: ./cached_jars
27+
key: ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-test-ftp
28+
restore-keys: |
29+
${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-test-ftp
30+
${{ runner.os }}-python-
31+
32+
- name: Build Worker Image
33+
uses: docker/build-push-action@v6
34+
with:
35+
context: .
36+
tags: mtsrus/syncmaster-worker:${{ github.sha }}
37+
target: test
38+
file: docker/Dockerfile.worker
39+
load: true
40+
cache-from: mtsrus/syncmaster-worker:develop
41+
42+
- name: Docker compose up
43+
run: |
44+
docker compose -f docker-compose.test.yml --profile all down -v --remove-orphans
45+
docker compose -f docker-compose.test.yml --profile ftp up -d --wait --wait-timeout 200
46+
env:
47+
WORKER_IMAGE_TAG: ${{ github.sha }}
48+
49+
- name: Run FTP Tests
50+
run: |
51+
docker compose -f ./docker-compose.test.yml --profile ftp exec -T worker coverage run -m pytest -vvv -s -m "worker and ftp"
52+
53+
- name: Dump worker logs on failure
54+
if: failure()
55+
uses: jwalton/gh-docker-logs@v2
56+
with:
57+
images: mtsrus/syncmaster-worker
58+
dest: ./logs
59+
60+
# This is important, as coverage is exported after receiving SIGTERM
61+
- name: Shutdown
62+
if: always()
63+
run: |
64+
docker compose -f docker-compose.test.yml --profile all down -v --remove-orphans
65+
66+
- name: Upload worker logs
67+
uses: actions/upload-artifact@v4
68+
if: failure()
69+
with:
70+
name: worker-logs-ftp
71+
path: logs/*
72+
73+
- name: Upload coverage results
74+
uses: actions/upload-artifact@v4
75+
with:
76+
name: coverage-ftp
77+
path: reports/*
78+
# https://github.com/actions/upload-artifact/issues/602
79+
include-hidden-files: true

‎.github/workflows/ftps-tests.yml‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
name: FTPS tests
2+
on:
3+
workflow_call:
4+
5+
env:
6+
DEFAULT_PYTHON: '3.12'
7+
8+
jobs:
9+
test:
10+
name: Run FTPS tests
11+
runs-on: ubuntu-latest
12+
13+
steps:
14+
- name: Checkout code
15+
uses: actions/checkout@v4
16+
17+
- name: Set up QEMU
18+
uses: docker/setup-qemu-action@v3
19+
20+
- name: Set up Docker Buildx
21+
uses: docker/setup-buildx-action@v3
22+
23+
- name: Cache jars
24+
uses: actions/cache@v4
25+
with:
26+
path: ./cached_jars
27+
key: ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-test-ftps
28+
restore-keys: |
29+
${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-test-ftps
30+
${{ runner.os }}-python-
31+
32+
- name: Build Worker Image
33+
uses: docker/build-push-action@v6
34+
with:
35+
context: .
36+
tags: mtsrus/syncmaster-worker:${{ github.sha }}
37+
target: test
38+
file: docker/Dockerfile.worker
39+
load: true
40+
cache-from: mtsrus/syncmaster-worker:develop
41+
42+
- name: Docker compose up
43+
run: |
44+
docker compose -f docker-compose.test.yml --profile all down -v --remove-orphans
45+
docker compose -f docker-compose.test.yml --profile ftps up -d --wait --wait-timeout 200
46+
env:
47+
WORKER_IMAGE_TAG: ${{ github.sha }}
48+
49+
- name: Run FTPS Tests
50+
run: |
51+
docker compose -f ./docker-compose.test.yml --profile ftps exec -T worker coverage run -m pytest -vvv -s -m "worker and ftps"
52+
53+
- name: Dump worker logs on failure
54+
if: failure()
55+
uses: jwalton/gh-docker-logs@v2
56+
with:
57+
images: mtsrus/syncmaster-worker
58+
dest: ./logs
59+
60+
# This is important, as coverage is exported after receiving SIGTERM
61+
- name: Shutdown
62+
if: always()
63+
run: |
64+
docker compose -f docker-compose.test.yml --profile all down -v --remove-orphans
65+
66+
- name: Upload worker logs
67+
uses: actions/upload-artifact@v4
68+
if: failure()
69+
with:
70+
name: worker-logs-ftps
71+
path: logs/*
72+
73+
- name: Upload coverage results
74+
uses: actions/upload-artifact@v4
75+
with:
76+
name: coverage-ftps
77+
path: reports/*
78+
# https://github.com/actions/upload-artifact/issues/602
79+
include-hidden-files: true

‎.github/workflows/tests.yml‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,14 @@ jobs:
4848
name: SFTP tests
4949
uses: ./.github/workflows/sftp-tests.yml
5050

51+
ftp_tests:
52+
name: FTP tests
53+
uses: ./.github/workflows/ftp-tests.yml
54+
55+
ftps_tests:
56+
name: FTPS tests
57+
uses: ./.github/workflows/ftps-tests.yml
58+
5159
scheduler_tests:
5260
name: Scheduler tests
5361
uses: ./.github/workflows/scheduler-tests.yml
@@ -60,7 +68,7 @@ jobs:
6068
name: Tests done
6169
runs-on: ubuntu-latest
6270

63-
needs: [unit_tests, scheduler_tests, oracle_tests, clickhouse_tests, mssql_tests, mysql_tests, hive_tests, hdfs_tests, s3_tests, sftp_tests]
71+
needs: [unit_tests, scheduler_tests, oracle_tests, clickhouse_tests, mssql_tests, mysql_tests, hive_tests, hdfs_tests, s3_tests, sftp_tests, ftp_tests, ftps_tests]
6472
steps:
6573
- name: Checkout code
6674
uses: actions/checkout@v4

‎Makefile‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,14 @@ test-integration-sftp: test-db ##@Test Run integration tests for SFTP
113113
docker compose -f docker-compose.test.yml --profile sftp up -d --wait $(DOCKER_COMPOSE_ARGS)
114114
${POETRY} run pytest ./tests/test_integration -m sftp $(PYTEST_ARGS)
115115

116+
test-integration-ftp: test-db ##@Test Run integration tests for FTP
117+
docker compose -f docker-compose.test.yml --profile ftp up -d --wait $(DOCKER_COMPOSE_ARGS)
118+
${POETRY} run pytest ./tests/test_integration -m ftp $(PYTEST_ARGS)
119+
120+
test-integration-ftps: test-db ##@Test Run integration tests for FTPS
121+
docker compose -f docker-compose.test.yml --profile ftps up -d --wait $(DOCKER_COMPOSE_ARGS)
122+
${POETRY} run pytest ./tests/test_integration -m ftps $(PYTEST_ARGS)
123+
116124
test-integration: test-db ##@Test Run all integration tests
117125
docker compose -f docker-compose.test.yml --profile all up -d --wait $(DOCKER_COMPOSE_ARGS)
118126
${POETRY} run pytest ./tests/test_integration $(PYTEST_ARGS)

‎README.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ List of currently supported connections:
4040
* MySQL
4141
* HDFS
4242
* S3
43+
* FTP
44+
* FTPS
4345
* SFTP
4446

4547
Current Data.SyncMaster implementation provides following components:

‎docker-compose.test.yml‎

Lines changed: 41 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ services:
125125
condition: service_completed_successfully
126126
rabbitmq:
127127
condition: service_healthy
128-
profiles: [worker, scheduler, s3, oracle, hdfs, hive, clickhouse, mysql, mssql, sftp, all]
128+
profiles: [worker, scheduler, s3, oracle, hdfs, hive, clickhouse, mysql, mssql, sftp, ftp, ftps, all]
129129

130130
test-postgres:
131131
image: postgres
@@ -139,7 +139,7 @@ services:
139139
interval: 30s
140140
timeout: 5s
141141
retries: 3
142-
profiles: [s3, oracle, clickhouse, mysql, mssql, hdfs, hive, sftp, all]
142+
profiles: [s3, oracle, clickhouse, mysql, mssql, hdfs, hive, sftp, ftp, ftps, all]
143143

144144
test-s3:
145145
image: bitnami/minio:latest
@@ -225,7 +225,7 @@ services:
225225
interval: 30s
226226
timeout: 5s
227227
retries: 3
228-
profiles: [hive, hdfs, s3, sftp, all]
228+
profiles: [hive, hdfs, s3, sftp, ftp, ftps, all]
229229

230230
keycloak:
231231
image: quay.io/keycloak/keycloak:latest
@@ -263,8 +263,8 @@ services:
263263
HIVE_METASTORE_DB_DRIVER: org.postgresql.Driver
264264
HIVE_METASTORE_DB_USER: test_hive
265265
HIVE_METASTORE_DB_PASSWORD: test_hive
266-
# writing spark dataframe to s3, sftp xml file fails without running hive metastore server
267-
profiles: [hive, hdfs, s3, sftp, all]
266+
# writing spark dataframe to s3, sftp, ftp, ftps xml file fails without running hive metastore server
267+
profiles: [hive, hdfs, s3, sftp, ftp, ftps, all]
268268

269269
test-sftp:
270270
image: ${SFTP_IMAGE:-linuxserver/openssh-server}
@@ -280,6 +280,42 @@ services:
280280
USER_PASSWORD: test_only
281281
profiles: [sftp, all]
282282

283+
test-ftp:
284+
image: ${FTP_IMAGE:-chonjay21/ftps:latest}
285+
restart: unless-stopped
286+
environment:
287+
- USE_SSL=false
288+
- PASSV_MIN_PORT=30000
289+
- PASSV_MAX_PORT=30010
290+
- APP_USER_NAME=syncmaster
291+
- APP_USER_PASSWD=test_only
292+
- APP_UID=1000
293+
- APP_GID=1000
294+
ports:
295+
- 2121:21
296+
- 30000-30010:30000-30010
297+
volumes:
298+
- ./docker/ftp/on_post_init.sh:/sources/ftp/eventscripts/on_post_init.sh
299+
profiles: [ftp, all]
300+
301+
test-ftps:
302+
image: ${FTPS_IMAGE:-chonjay21/ftps:latest}
303+
restart: unless-stopped
304+
environment:
305+
- USE_SSL=true
306+
- PASSV_MIN_PORT=30020
307+
- PASSV_MAX_PORT=30030
308+
- APP_USER_NAME=syncmaster
309+
- APP_USER_PASSWD=test_only
310+
- APP_UID=1000
311+
- APP_GID=1000
312+
ports:
313+
- 2122:21
314+
- 30020-30030:30020-30030
315+
volumes:
316+
- ./docker/ftp/on_post_init.sh:/sources/ftps/eventscripts/on_post_init.sh
317+
profiles: [ftps, all]
318+
283319
volumes:
284320
postgres_test_data:
285321
rabbitmq_test_data:

‎docker/ftp/on_post_init.sh‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/usr/bin/env bash
2+
set -e
3+
4+
# https://serverfault.com/questions/157159/too-many-ftp-connection-causing-error-421
5+
echo "max_per_ip=0" >> /etc/vsftpd/vsftpd.conf
6+
7+
# https://serverfault.com/questions/65002/vsftpd-and-implicit-ssl
8+
echo "implicit_ssl=NO" >> /etc/vsftpd/vsftpd.conf
9+
10+
# enable anonymous login for both FTP and FTPS
11+
echo "anonymous_enable=YES" >> /etc/vsftpd/vsftpd.conf
12+
echo "allow_anon_ssl=YES" >> /etc/vsftpd/vsftpd.conf

‎docs/changelog/next_release/191.feature.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Add logic for handling FTP|FTPS transfers

0 commit comments

Comments
 (0)