Skip to content

Commit 8e634e9

Browse files
dolfinusdolfinus
committed
Merge branch 'develop'
2 parents 9083521 + 4b487d1 commit 8e634e9

File tree

522 files changed

+30224
-9902
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

522 files changed

+30224
-9902
lines changed

‎.dockerignore‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
1-
__pycache__/
2-
*.py[cod]
1+
**/__pycache__/
2+
**.pyc
3+
**.pyo
4+
**.pyd
35
*.envx
46
*.cover
57
.coverage

‎.env.docker‎

Lines changed: 47 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -1,53 +1,50 @@
11
TZ=UTC
2-
3-
# Postgres
4-
POSTGRES_HOST=db
5-
POSTGRES_PORT=5432
6-
POSTGRES_DB=syncmaster
7-
POSTGRES_USER=user
8-
POSTGRES_PASSWORD=secret
9-
10-
# RabbitMQ
11-
RABBITMQ_HOST=rabbitmq
12-
RABBITMQ_PORT=5672
13-
RABBITMQ_USER=guest
14-
RABBITMQ_PASSWORD=guest
15-
16-
# HDFS
17-
HDFS_HOST=test-hive
18-
HDFS_WEBHDFS_PORT=9870
19-
HDFS_IPC_PORT=9820
20-
21-
# S3
22-
MINIO_ACCESS_KEY=syncmaster
23-
MINIO_ROOT_USER=syncmaster
24-
MINIO_SECRET_KEY=a2b6ad842d16f47beaa077b0e50c217a3bded54c
25-
MINIO_ROOT_PASSWORD=a2b6ad842d16f47beaa077b0e50c217a3bded54c
26-
27-
TEST_S3_HOST=test-s3
28-
TEST_S3_PORT=9000
29-
TEST_S3_ACCESS_KEY=syncmaster
30-
TEST_S3_SECRET_KEY=a2b6ad842d16f47beaa077b0e50c217a3bded54c
31-
TEST_S3_BUCKET=syncmaster
32-
33-
TEST_POSTGRES_HOST=test-postgres
34-
TEST_POSTGRES_PORT=5432
35-
TEST_POSTGRES_USER=user
36-
TEST_POSTGRES_PASSWORD=secret
37-
TEST_POSTGRES_DB=syncmaster
38-
39-
TEST_ORACLE_HOST=test-oracle
40-
TEST_ORACLE_PORT=1521
41-
TEST_ORACLE_USER=test_user
42-
TEST_ORACLE_PASSWORD=test_password
43-
TEST_ORACLE_SERVICE_NAME=XEPDB1
44-
45-
TEST_HIVE_CLUSTER=test-hive
46-
TEST_HIVE_USER=hive_user
47-
TEST_HIVE_PASSWORD=hive_password
48-
492
ENV=LOCAL
503

51-
SPARK_CONF_DIR=/app/tests/spark/hive/conf/
52-
HADOOP_CONF_DIR=/app/tests/spark/hadoop/
53-
HIVE_CONF_DIR=/app/tests/spark/hive/conf/
4+
# Logging options
5+
SYNCMASTER__LOGGING__SETUP=True
6+
SYNCMASTER__LOGGING__PRESET=colored
7+
8+
# Common DB options
9+
SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster
10+
11+
# Encrypt / Decrypt credentials data using this Fernet key.
12+
# !!! GENERATE YOUR OWN COPY FOR PRODUCTION USAGE !!!
13+
SYNCMASTER__ENCRYPTION__SECRET_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
14+
15+
# Common RabbitMQ options
16+
SYNCMASTER__BROKER__URL=amqp://guest:guest@rabbitmq:5672
17+
18+
# Server options
19+
SYNCMASTER__SERVER__SESSION__SECRET_KEY=generate_some_random_string
20+
# !!! NEVER USE ON PRODUCTION !!!
21+
SYNCMASTER__SERVER__DEBUG=true
22+
23+
# Keycloak Auth
24+
#SYNCMASTER__AUTH__PROVIDER=syncmaster.server.providers.auth.keycloak_provider.KeycloakAuthProvider
25+
SYNCMASTER__AUTH__KEYCLOAK__SERVER_URL=http://keycloak:8080
26+
SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME=manually_created
27+
SYNCMASTER__AUTH__KEYCLOAK__CLIENT_ID=manually_created
28+
SYNCMASTER__AUTH__KEYCLOAK__CLIENT_SECRET=generated_by_keycloak
29+
SYNCMASTER__AUTH__KEYCLOAK__REDIRECT_URI=http://localhost:8000/auth/callback
30+
SYNCMASTER__AUTH__KEYCLOAK__SCOPE=email
31+
SYNCMASTER__AUTH__KEYCLOAK__VERIFY_SSL=False
32+
33+
# Dummy Auth
34+
SYNCMASTER__AUTH__PROVIDER=syncmaster.server.providers.auth.dummy_provider.DummyAuthProvider
35+
SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=generate_another_random_string
36+
37+
# Scheduler options
38+
SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200
39+
40+
# Worker options
41+
SYNCMASTER__WORKER__LOG_URL_TEMPLATE=https://logs.location.example.com/syncmaster-worker?correlation_id={{ correlation_id }}&run_id={{ run.id }}
42+
SYNCMASTER__HWM_STORE__ENABLED=true
43+
SYNCMASTER__HWM_STORE__TYPE=horizon
44+
SYNCMASTER__HWM_STORE__URL=http://horizon:8000
45+
SYNCMASTER__HWM_STORE__NAMESPACE=syncmaster_namespace
46+
SYNCMASTER__HWM_STORE__USER=admin
47+
SYNCMASTER__HWM_STORE__PASSWORD=123UsedForTestOnly@!
48+
49+
# Frontend options
50+
SYNCMASTER__UI__API_BROWSER_URL=http://localhost:8000

‎.env.docker.test‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
# Tests-only
2+
TEST_S3_HOST_FOR_CONFTEST=test-s3
3+
TEST_S3_PORT_FOR_CONFTEST=9000
4+
TEST_S3_HOST_FOR_WORKER=test-s3
5+
TEST_S3_PORT_FOR_WORKER=9000
6+
TEST_S3_ACCESS_KEY=syncmaster
7+
TEST_S3_SECRET_KEY=123UsedForTestOnly@!
8+
TEST_S3_BUCKET=syncmaster
9+
10+
TEST_POSTGRES_HOST_FOR_CONFTEST=test-postgres
11+
TEST_POSTGRES_PORT_FOR_CONFTEST=5432
12+
TEST_POSTGRES_HOST_FOR_WORKER=test-postgres
13+
TEST_POSTGRES_PORT_FOR_WORKER=5432
14+
TEST_POSTGRES_USER=syncmaster
15+
TEST_POSTGRES_PASSWORD=123UsedForTestOnly@!
16+
TEST_POSTGRES_DB=syncmaster
17+
18+
TEST_ORACLE_HOST_FOR_CONFTEST=test-oracle
19+
TEST_ORACLE_PORT_FOR_CONFTEST=1521
20+
TEST_ORACLE_HOST_FOR_WORKER=test-oracle
21+
TEST_ORACLE_PORT_FOR_WORKER=1521
22+
TEST_ORACLE_USER=syncmaster
23+
TEST_ORACLE_PASSWORD=123UsedForTestOnly@!
24+
TEST_ORACLE_SERVICE_NAME=XEPDB1
25+
26+
TEST_CLICKHOUSE_HOST_FOR_CONFTEST=test-clickhouse
27+
TEST_CLICKHOUSE_PORT_FOR_CONFTEST=8123
28+
TEST_CLICKHOUSE_HOST_FOR_WORKER=test-clickhouse
29+
TEST_CLICKHOUSE_PORT_FOR_WORKER=8123
30+
TEST_CLICKHOUSE_USER=syncmaster
31+
TEST_CLICKHOUSE_PASSWORD=123UsedForTestOnly@!
32+
TEST_CLICKHOUSE_DB=syncmaster
33+
34+
TEST_MSSQL_HOST_FOR_CONFTEST=test-mssql
35+
TEST_MSSQL_PORT_FOR_CONFTEST=1433
36+
TEST_MSSQL_HOST_FOR_WORKER=test-mssql
37+
TEST_MSSQL_PORT_FOR_WORKER=1433
38+
TEST_MSSQL_USER=syncmaster
39+
TEST_MSSQL_PASSWORD=123UsedForTestOnly@!
40+
TEST_MSSQL_DB=syncmaster
41+
42+
TEST_MYSQL_HOST_FOR_CONFTEST=test-mysql
43+
TEST_MYSQL_PORT_FOR_CONFTEST=3306
44+
TEST_MYSQL_HOST_FOR_WORKER=test-mysql
45+
TEST_MYSQL_PORT_FOR_WORKER=3306
46+
TEST_MYSQL_USER=syncmaster
47+
TEST_MYSQL_PASSWORD=123UsedForTestOnly@!
48+
TEST_MYSQL_DB=syncmaster
49+
50+
TEST_HIVE_CLUSTER=test-hive
51+
TEST_HIVE_USER=syncmaster
52+
TEST_HIVE_PASSWORD=123UsedForTestOnly@!
53+
54+
TEST_HDFS_HOST=test-hive
55+
TEST_HDFS_WEBHDFS_PORT=9870
56+
TEST_HDFS_IPC_PORT=9820
57+
58+
TEST_SFTP_HOST_FOR_CONFTEST=test-sftp
59+
TEST_SFTP_PORT_FOR_CONFTEST=2222
60+
TEST_SFTP_HOST_FOR_WORKER=test-sftp
61+
TEST_SFTP_PORT_FOR_WORKER=2222
62+
TEST_SFTP_USER=syncmaster
63+
TEST_SFTP_PASSWORD=123UsedForTestOnly@!
64+
65+
TEST_FTP_HOST_FOR_CONFTEST=test-ftp
66+
TEST_FTP_PORT_FOR_CONFTEST=21
67+
TEST_FTP_HOST_FOR_WORKER=test-ftp
68+
TEST_FTP_PORT_FOR_WORKER=21
69+
TEST_FTP_USER=syncmaster
70+
TEST_FTP_PASSWORD=123UsedForTestOnly@!
71+
72+
TEST_FTPS_HOST_FOR_CONFTEST=test-ftps
73+
TEST_FTPS_PORT_FOR_CONFTEST=21
74+
TEST_FTPS_HOST_FOR_WORKER=test-ftps
75+
TEST_FTPS_PORT_FOR_WORKER=21
76+
TEST_FTPS_USER=syncmaster
77+
TEST_FTPS_PASSWORD=123UsedForTestOnly@!
78+
79+
TEST_SAMBA_HOST_FOR_CONFTEST=test-samba
80+
TEST_SAMBA_PORT_FOR_CONFTEST=445
81+
TEST_SAMBA_HOST_FOR_WORKER=test-samba
82+
TEST_SAMBA_PORT_FOR_WORKER=445
83+
TEST_SAMBA_DOMAIN=domain
84+
TEST_SAMBA_PROTOCOL=SMB
85+
TEST_SAMBA_UID=1000
86+
TEST_SAMBA_GID=1000
87+
TEST_SAMBA_SHARE=SmbShare
88+
TEST_SAMBA_USER=syncmaster
89+
TEST_SAMBA_PASSWORD=123UsedForTestOnly@!
90+
TEST_SAMBA_AUTH_TYPE=NTLMv2
91+
92+
TEST_WEBDAV_HOST_FOR_CONFTEST=test-webdav
93+
TEST_WEBDAV_PORT_FOR_CONFTEST=80
94+
TEST_WEBDAV_HOST_FOR_WORKER=test-webdav
95+
TEST_WEBDAV_PORT_FOR_WORKER=80
96+
TEST_WEBDAV_USER=syncmaster
97+
TEST_WEBDAV_PASSWORD=123UsedForTestOnly@!
98+
99+
SPARK_CONF_DIR=/app/tests/spark/hive/conf/
100+
HADOOP_CONF_DIR=/app/tests/spark/hadoop/
101+
HIVE_CONF_DIR=/app/tests/spark/hive/conf/

‎.env.local‎

Lines changed: 47 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -1,53 +1,50 @@
11
export TZ=UTC
2-
3-
# Postgres
4-
export POSTGRES_HOST=localhost
5-
export POSTGRES_PORT=5432
6-
export POSTGRES_DB=syncmaster
7-
export POSTGRES_USER=user
8-
export POSTGRES_PASSWORD=secret
9-
10-
# RabbitMQ
11-
export RABBITMQ_HOST=localhost
12-
export RABBITMQ_PORT=5672
13-
export RABBITMQ_USER=guest
14-
export RABBITMQ_PASSWORD=guest
15-
16-
# HDFS
17-
export HDFS_HOST=localhost
18-
export HDFS_WEBHDFS_PORT=9870
19-
export HDFS_IPC_PORT=9820
20-
21-
# S3
22-
export MINIO_ACCESS_KEY=syncmaster
23-
export MINIO_ROOT_USER=syncmaster
24-
export MINIO_SECRET_KEY=a2b6ad842d16f47beaa077b0e50c217a3bded54c
25-
export MINIO_ROOT_PASSWORD=a2b6ad842d16f47beaa077b0e50c217a3bded54c
26-
27-
export TEST_S3_HOST=localhost
28-
export TEST_S3_PORT=9010
29-
export TEST_S3_ACCESS_KEY=syncmaster
30-
export TEST_S3_SECRET_KEY=a2b6ad842d16f47beaa077b0e50c217a3bded54c
31-
export TEST_S3_BUCKET=syncmaster
32-
33-
export TEST_POSTGRES_HOST=localhost
34-
export TEST_POSTGRES_PORT=5433
35-
export TEST_POSTGRES_USER=user
36-
export TEST_POSTGRES_PASSWORD=secret
37-
export TEST_POSTGRES_DB=syncmaster
38-
39-
export TEST_ORACLE_HOST=localhost
40-
export TEST_ORACLE_PORT=1522
41-
export TEST_ORACLE_USER=test_user
42-
export TEST_ORACLE_PASSWORD=test_password
43-
export TEST_ORACLE_SERVICE_NAME=XEPDB1
44-
45-
export TEST_HIVE_CLUSTER=test-hive
46-
export TEST_HIVE_USER=hive_user
47-
export TEST_HIVE_PASSWORD=hive_password
48-
492
export ENV=LOCAL
503

51-
export SPARK_CONF_DIR=./app/tests/spark/hive/conf/
52-
export HADOOP_CONF_DIR=./app/tests/spark/hadoop/
53-
export HIVE_CONF_DIR=./app/tests/spark/hive/conf/
4+
# Logging options
5+
export SYNCMASTER__LOGGING__SETUP=True
6+
export SYNCMASTER__LOGGING__PRESET=colored
7+
8+
# Common DB options
9+
export SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@localhost:5432/syncmaster
10+
11+
# Encrypt / Decrypt credentials data using this Fernet key.
12+
# !!! GENERATE YOUR OWN COPY FOR PRODUCTION USAGE !!!
13+
export SYNCMASTER__ENCRYPTION__SECRET_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
14+
15+
# Common RabbitMQ options
16+
export SYNCMASTER__BROKER__URL=amqp://guest:guest@localhost:5672
17+
18+
# Server options
19+
export SYNCMASTER__SERVER__SESSION__SECRET_KEY=generate_some_random_string
20+
# !!! NEVER USE ON PRODUCTION !!!
21+
export SYNCMASTER__SERVER__DEBUG=true
22+
23+
# Keycloak Auth
24+
#export SYNCMASTER__AUTH__PROVIDER=syncmaster.server.providers.auth.keycloak_provider.KeycloakAuthProvider
25+
export SYNCMASTER__AUTH__KEYCLOAK__SERVER_URL=http://localhost:8080
26+
export SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME=manually_created
27+
export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_ID=manually_created
28+
export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_SECRET=generated_by_keycloak
29+
export SYNCMASTER__AUTH__KEYCLOAK__REDIRECT_URI=http://localhost:8000/auth/callback
30+
export SYNCMASTER__AUTH__KEYCLOAK__SCOPE=email
31+
export SYNCMASTER__AUTH__KEYCLOAK__VERIFY_SSL=False
32+
33+
# Dummy Auth
34+
export SYNCMASTER__AUTH__PROVIDER=syncmaster.server.providers.auth.dummy_provider.DummyAuthProvider
35+
export SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=generate_another_random_string
36+
37+
# Scheduler options
38+
export SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200
39+
40+
# Worker options
41+
export SYNCMASTER__WORKER__LOG_URL_TEMPLATE="https://logs.location.example.com/syncmaster-worker?correlation_id={{ correlation_id }}&run_id={{ run.id }}"
42+
export SYNCMASTER__HWM_STORE__ENABLED=true
43+
export SYNCMASTER__HWM_STORE__TYPE=horizon
44+
export SYNCMASTER__HWM_STORE__URL=http://localhost:8020
45+
export SYNCMASTER__HWM_STORE__NAMESPACE=syncmaster_namespace
46+
export SYNCMASTER__HWM_STORE__USER=admin
47+
export SYNCMASTER__HWM_STORE__PASSWORD=123UsedForTestOnly@!
48+
49+
# Frontend options
50+
export SYNCMASTER__UI__API_BROWSER_URL=http://localhost:8000

0 commit comments

Comments
 (0)