Skip to content

Commit b448bbd

Browse files
dolfinusdolfinus
authored
Merge pull request #126 from MobileTeleSystems/develop
Release 0.9.0
2 parents 4ac9e3b + 9c42fc1 commit b448bbd

File tree

359 files changed

+17460
-3849
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

359 files changed

+17460
-3849
lines changed

‎.env.dependencies‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,21 @@ ALLOW_PLAINTEXT_LISTENER=yes
55
KAFKA_ENABLE_KRAFT=no
66
KAFKA_CLIENT_USERS=onetl
77
KAFKA_CLIENT_PASSWORDS=uufoFae9sahSoidoo0eagaidaoreif6z
8+
KAFKA_ZOOKEEPER_BOOTSTRAP_SCRAM_USERS=true
89
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true
910
KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
10-
KAFKA_CFG_LISTENERS=PLAINTEXT://:9092
11+
KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL_PLAINTEXT_ANONYMOUS
12+
KAFKA_CFG_LISTENERS=INTERNAL_PLAINTEXT_ANONYMOUS://:9092,EXTERNAL_PLAINTEXT_ANONYMOUS://:9093,INTERNAL_PLAINTEXT_SASL://:9094,EXTERNAL_PLAINTEXT_SASL://:9095
13+
KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL_PLAINTEXT_ANONYMOUS://kafka:9092,EXTERNAL_PLAINTEXT_ANONYMOUS://localhost:9093,INTERNAL_PLAINTEXT_SASL://kafka:9094,EXTERNAL_PLAINTEXT_SASL://localhost:9095
14+
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL_PLAINTEXT_ANONYMOUS:PLAINTEXT,EXTERNAL_PLAINTEXT_ANONYMOUS:PLAINTEXT,INTERNAL_PLAINTEXT_SASL:SASL_PLAINTEXT,EXTERNAL_PLAINTEXT_SASL:SASL_PLAINTEXT
15+
KAFKA_CFG_SASL_ENABLED_MECHANISMS=PLAIN,SCRAM-SHA-256,SCRAM-SHA-512
1116
# old config names for <1.1.1
1217
KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
13-
KAFKA_LISTENERS=PLAINTEXT://:9092
18+
KAFKA_INTER_BROKER_LISTENER_NAME=INTERNAL_PLAINTEXT_ANONYMOUS
19+
KAFKA_LISTENERS=INTERNAL_PLAINTEXT_ANONYMOUS://:9092,EXTERNAL_PLAINTEXT_ANONYMOUS://:9093,INTERNAL_PLAINTEXT_SASL://:9094,EXTERNAL_PLAINTEXT_SASL://:9095
20+
KAFKA_ADVERTISED_LISTENERS=INTERNAL_PLAINTEXT_ANONYMOUS://kafka:9092,EXTERNAL_PLAINTEXT_ANONYMOUS://localhost:9093,INTERNAL_PLAINTEXT_SASL://kafka:9094,EXTERNAL_PLAINTEXT_SASL://localhost:9095
21+
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL_PLAINTEXT_ANONYMOUS:PLAINTEXT,EXTERNAL_PLAINTEXT_ANONYMOUS:PLAINTEXT,INTERNAL_PLAINTEXT_SASL:SASL_PLAINTEXT,EXTERNAL_PLAINTEXT_SASL:SASL_PLAINTEXT
22+
KAFKA_SASL_ENABLED_MECHANISMS=PLAIN,SCRAM-SHA-256,SCRAM-SHA-512
1423

1524
# Mongo
1625
MONGO_INITDB_ROOT_USERNAME=onetl

‎.env.docker‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,8 @@ ONETL_CH_PASSWORD=
1717

1818
# Kafka
1919
ONETL_KAFKA_HOST=kafka
20-
ONETL_KAFKA_PORT=9092
20+
ONETL_KAFKA_PLAINTEXT_ANONYMOUS_PORT=9092
21+
ONETL_KAFKA_PLAINTEXT_SASL_PORT=9094
2122
ONETL_KAFKA_USER=onetl
2223
ONETL_KAFKA_PASSWORD=uufoFae9sahSoidoo0eagaidaoreif6z
2324

@@ -77,7 +78,8 @@ ONETL_S3_BUCKET=onetl
7778

7879
# HDFS
7980
ONETL_HDFS_HOST=hdfs
80-
ONETL_HDFS_PORT=9870
81+
ONETL_HDFS_WEBHDFS_PORT=9870
82+
ONETL_HDFS_IPC_PORT=9820
8183

8284
# SFTP
8385
ONETL_SFTP_HOST=sftp

‎.env.local‎

Lines changed: 66 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -1,92 +1,94 @@
1-
TZ=UTC
1+
export TZ=UTC
22

33
# Greenplum
4-
ONETL_GP_HOST=localhost
5-
ONETL_GP_PORT=5433
6-
ONETL_GP_DATABASE=testdb
7-
ONETL_GP_USER=tester
8-
ONETL_GP_PASSWORD=pivotal
4+
export ONETL_GP_HOST=localhost
5+
export ONETL_GP_PORT=5433
6+
export ONETL_GP_DATABASE=testdb
7+
export ONETL_GP_USER=tester
8+
export ONETL_GP_PASSWORD=pivotal
99

1010
# ClickHouse
11-
ONETL_CH_HOST=localhost
12-
ONETL_CH_PORT=8123
13-
ONETL_CH_PORT_CLIENT=9001
14-
ONETL_CH_DATABASE=default
15-
ONETL_CH_USER=default
16-
ONETL_CH_PASSWORD=
11+
export ONETL_CH_HOST=localhost
12+
export ONETL_CH_PORT=8123
13+
export ONETL_CH_PORT_CLIENT=9001
14+
export ONETL_CH_DATABASE=default
15+
export ONETL_CH_USER=default
16+
export ONETL_CH_PASSWORD=
1717

1818
# Kafka
19-
ONETL_KAFKA_HOST=localhost
20-
ONETL_KAFKA_PORT=9092
21-
ONETL_KAFKA_USER=onetl
22-
ONETL_KAFKA_PASSWORD=uufoFae9sahSoidoo0eagaidaoreif6z
19+
export ONETL_KAFKA_HOST=localhost
20+
export ONETL_KAFKA_PLAINTEXT_ANONYMOUS_PORT=9093
21+
export ONETL_KAFKA_PLAINTEXT_SASL_PORT=9095
22+
export ONETL_KAFKA_USER=onetl
23+
export ONETL_KAFKA_PASSWORD=uufoFae9sahSoidoo0eagaidaoreif6z
2324

2425
# Mongo
25-
ONETL_MONGO_HOST=localhost
26-
ONETL_MONGO_PORT=27017
27-
ONETL_MONGO_DB=admin
28-
ONETL_MONGO_USER=onetl
29-
ONETL_MONGO_PASSWORD=E4j7h!9A
26+
export ONETL_MONGO_HOST=localhost
27+
export ONETL_MONGO_PORT=27017
28+
export ONETL_MONGO_DB=admin
29+
export ONETL_MONGO_USER=onetl
30+
export ONETL_MONGO_PASSWORD=E4j7h!9A
3031

3132
# MSSQL
32-
ONETL_MSSQL_HOST=localhost
33-
ONETL_MSSQL_PORT=1433
34-
ONETL_MSSQL_DATABASE=onetl
35-
ONETL_MSSQL_USER=onetl
36-
ONETL_MSSQL_PASSWORD=7ellowEl7akey
33+
export ONETL_MSSQL_HOST=localhost
34+
export ONETL_MSSQL_PORT=1433
35+
export ONETL_MSSQL_DATABASE=onetl
36+
export ONETL_MSSQL_USER=onetl
37+
export ONETL_MSSQL_PASSWORD=7ellowEl7akey
3738

3839
# MySQL
39-
ONETL_MYSQL_HOST=localhost
40-
ONETL_MYSQL_PORT=3306
41-
ONETL_MYSQL_DATABASE=onetl
42-
ONETL_MYSQL_USER=onetl
43-
ONETL_MYSQL_PASSWORD=ohbuz9Eochaj9saibooK3thooGa5aesh
40+
export ONETL_MYSQL_HOST=localhost
41+
export ONETL_MYSQL_PORT=3306
42+
export ONETL_MYSQL_DATABASE=onetl
43+
export ONETL_MYSQL_USER=onetl
44+
export ONETL_MYSQL_PASSWORD=ohbuz9Eochaj9saibooK3thooGa5aesh
4445

4546
# Postgres
46-
ONETL_PG_HOST=localhost
47-
ONETL_PG_PORT=5432
48-
ONETL_PG_DATABASE=onetl
49-
ONETL_PG_USER=onetl
50-
ONETL_PG_PASSWORD=ohtae0luxeshi1uraeluMoh9IShah7ai
47+
export ONETL_PG_HOST=localhost
48+
export ONETL_PG_PORT=5432
49+
export ONETL_PG_DATABASE=onetl
50+
export ONETL_PG_USER=onetl
51+
export ONETL_PG_PASSWORD=ohtae0luxeshi1uraeluMoh9IShah7ai
5152

5253
# Oracle
53-
ONETL_ORA_HOST=localhost
54-
ONETL_ORA_PORT=1522
55-
ONETL_ORA_SERVICE_NAME=XEPDB1
56-
ONETL_ORA_USER=onetl
57-
ONETL_ORA_PASSWORD=Yoequ2Hoeceit4ch
54+
export ONETL_ORA_HOST=localhost
55+
export ONETL_ORA_PORT=1522
56+
export ONETL_ORA_SERVICE_NAME=XEPDB1
57+
export ONETL_ORA_USER=onetl
58+
export ONETL_ORA_PASSWORD=Yoequ2Hoeceit4ch
5859

5960
# FTP
60-
ONETL_FTP_HOST=localhost
61-
ONETL_FTP_PORT=2121
62-
ONETL_FTP_USER=onetl
63-
ONETL_FTP_PASSWORD=awd123fd1
61+
export ONETL_FTP_HOST=localhost
62+
export ONETL_FTP_PORT=2121
63+
export ONETL_FTP_USER=onetl
64+
export ONETL_FTP_PASSWORD=awd123fd1
6465

6566
# FTPS
66-
ONETL_FTPS_HOST=localhost
67-
ONETL_FTPS_PORT=2122
68-
ONETL_FTPS_USER=onetl
69-
ONETL_FTPS_PASSWORD=awd123fd1
67+
export ONETL_FTPS_HOST=localhost
68+
export ONETL_FTPS_PORT=2122
69+
export ONETL_FTPS_USER=onetl
70+
export ONETL_FTPS_PASSWORD=awd123fd1
7071

7172
# S3
72-
ONETL_S3_HOST=localhost
73-
ONETL_S3_PORT=9010
74-
ONETL_S3_ACCESS_KEY=onetl
75-
ONETL_S3_SECRET_KEY=woh3fogh3Biereu3quee1aidu9theiro
76-
ONETL_S3_BUCKET=onetl
73+
export ONETL_S3_HOST=localhost
74+
export ONETL_S3_PORT=9010
75+
export ONETL_S3_ACCESS_KEY=onetl
76+
export ONETL_S3_SECRET_KEY=woh3fogh3Biereu3quee1aidu9theiro
77+
export ONETL_S3_BUCKET=onetl
7778

7879
# HDFS
79-
ONETL_HDFS_HOST=localhost
80-
ONETL_HDFS_PORT=9870
80+
export ONETL_HDFS_HOST=localhost
81+
export ONETL_HDFS_WEBHDFS_PORT=9870
82+
export ONETL_HDFS_IPC_PORT=9820
8183

8284
# SFTP
83-
ONETL_SFTP_HOST=localhost
84-
ONETL_SFTP_PORT=2222
85-
ONETL_SFTP_USER=onetl
86-
ONETL_SFTP_PASSWORD=AesujeifohgoaCu0Boosiet5aimeitho
85+
export ONETL_SFTP_HOST=localhost
86+
export ONETL_SFTP_PORT=2222
87+
export ONETL_SFTP_USER=onetl
88+
export ONETL_SFTP_PASSWORD=AesujeifohgoaCu0Boosiet5aimeitho
8789

8890
# Webdav
89-
ONETL_WEBDAV_HOST=localhost
90-
ONETL_WEBDAV_PORT=8000
91-
ONETL_WEBDAV_USER=onetl
92-
ONETL_WEBDAV_PASSWORD=awd123fd1
91+
export ONETL_WEBDAV_HOST=localhost
92+
export ONETL_WEBDAV_PORT=8000
93+
export ONETL_WEBDAV_USER=onetl
94+
export ONETL_WEBDAV_PASSWORD=awd123fd1

‎.github/workflows/data/clickhouse/matrix.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ min: &min
77
max: &max
88
spark-version: 3.4.1
99
python-version: '3.11'
10-
java-version: 17
10+
java-version: 20
1111
os: ubuntu-latest
1212

1313
latest: &latest

‎.github/workflows/data/core/matrix.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ min: &min
77
max: &max
88
spark-version: 3.4.1
99
python-version: '3.11'
10-
java-version: 17
10+
java-version: 20
1111
os: ubuntu-latest
1212

1313
latest: &latest

‎.github/workflows/data/file-df/ignored.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
docs

‎.github/workflows/data/file-df/tracked.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
.github/workflows/data/file-df/**
2+
onetl/file_df_connection/spark_file_df_connection.py
3+
onetl/file/file_df_reader/**
4+
onetl/file/file_df_writer/**
5+
onetl/file/__init__.py
6+
tests/resources/file_df_connection/**

‎.github/workflows/data/file/tracked.txt‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,11 @@
11
.github/workflows/data/file/**
22
onetl/file_connection/file_connection.py
3-
onetl/file/**
3+
onetl/file/file_downloader/**
4+
onetl/file/file_mover/**
5+
onetl/file/file_uploader/**
6+
onetl/file/filter/**
7+
onetl/file/limit/**
8+
onetl/file/file_result.py
9+
onetl/file/file_set.py
10+
onetl/file/__init__.py
11+
tests/resources/file_connection/**

‎.github/workflows/data/hdfs/matrix.yml‎

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,30 @@
11
min: &min
2+
hadoop-version: hadoop2-hdfs
3+
spark-version: 2.3.1
24
python-version: '3.7'
5+
java-version: 8
36
os: ubuntu-latest
47

58
max: &max
9+
hadoop-version: hadoop3-hdfs
10+
spark-version: 3.4.1
611
python-version: '3.11'
12+
java-version: 20
13+
os: ubuntu-latest
14+
15+
latest: &latest
16+
hadoop-version: hadoop3-hdfs
17+
spark-version: latest
18+
python-version: '3.11'
19+
java-version: 20
720
os: ubuntu-latest
821

922
matrix:
1023
small:
11-
- hadoop-version: hadoop3-hdfs
12-
<<: *max
13-
full: &full
14-
- hadoop-version: hadoop2-hdfs
15-
<<: *min
16-
- hadoop-version: hadoop3-hdfs
17-
<<: *max
18-
nightly: *full
24+
- *max
25+
full:
26+
- *min
27+
- *max
28+
nightly:
29+
- *min
30+
- *latest

‎.github/workflows/data/hdfs/tracked.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
**/hdfs*
1+
**/*hdfs*
22
onetl/connection/kerberos_helpers.py
33
docker-compose.yml
44
.env.dependencies

0 commit comments

Comments
 (0)