Merge branch 'develop'

Author: dolfinus

.env.dependencies

@@ -1,79 +1,87 @@
 TZ=UTC
+SPARK_EXTERNAL_IP=onetl
 
 # Greenplum
 ONETL_GP_HOST=greenplum
 ONETL_GP_PORT=5432
-ONETL_GP_DATABASE=postgres
-ONETL_GP_USER=gpadmin
-ONETL_GP_PASSWORD=
+ONETL_GP_DATABASE=onetl
+ONETL_GP_USER=onetl
+ONETL_GP_PASSWORD=123UsedForTestOnly@!
 
 # ClickHouse
 ONETL_CH_HOST=clickhouse
 ONETL_CH_PORT=8123
 ONETL_CH_PORT_CLIENT=9000
 ONETL_CH_DATABASE=default
 ONETL_CH_USER=default
-ONETL_CH_PASSWORD=
+ONETL_CH_PASSWORD=123UsedForTestOnly@!
 
 # Kafka
 ONETL_KAFKA_HOST=kafka
 ONETL_KAFKA_PLAINTEXT_ANONYMOUS_PORT=9092
 ONETL_KAFKA_PLAINTEXT_SASL_PORT=9094
 ONETL_KAFKA_USER=onetl
-ONETL_KAFKA_PASSWORD=uufoFae9sahSoidoo0eagaidaoreif6z
+ONETL_KAFKA_PASSWORD=123UsedForTestOnly@!
 
 # Mongo
 ONETL_MONGO_HOST=mongodb
 ONETL_MONGO_PORT=27017
-ONETL_MONGO_DB=admin
+ONETL_MONGO_DB=onetl
 ONETL_MONGO_USER=onetl
-ONETL_MONGO_PASSWORD=E4j7h!9A
+ONETL_MONGO_PASSWORD=123UsedForTestOnly@!
+ONETL_MONGO_ROOT_DB=admin
+ONETL_MONGO_ROOT_USER=admin
+ONETL_MONGO_ROOT_PASSWORD=123AdminUserForTests@!
 
 # MSSQL
 ONETL_MSSQL_HOST=mssql
 ONETL_MSSQL_PORT=1433
 ONETL_MSSQL_DATABASE=onetl
 ONETL_MSSQL_USER=onetl
-ONETL_MSSQL_PASSWORD=7ellowEl7akey
+ONETL_MSSQL_PASSWORD=123UsedForTestOnly@!
 
 # MySQL
 ONETL_MYSQL_HOST=mysql
 ONETL_MYSQL_PORT=3306
 ONETL_MYSQL_DATABASE=onetl
 ONETL_MYSQL_USER=onetl
-ONETL_MYSQL_PASSWORD=ohbuz9Eochaj9saibooK3thooGa5aesh
+ONETL_MYSQL_PASSWORD=123UsedForTestOnly@!
+ONETL_MYSQL_ROOT_USER=root
+ONETL_MYSQL_ROOT_PASSWORD=123RootUserForTestsOnly@!
 
 # Postgres
 ONETL_PG_HOST=postgres
 ONETL_PG_PORT=5432
 ONETL_PG_DATABASE=onetl
 ONETL_PG_USER=onetl
-ONETL_PG_PASSWORD=ohtae0luxeshi1uraeluMoh9IShah7ai
+ONETL_PG_PASSWORD=123UsedForTestOnly@!
 
 # Oracle
 ONETL_ORA_HOST=oracle
 ONETL_ORA_PORT=1521
 ONETL_ORA_SERVICE_NAME=FREEPDB1
 ONETL_ORA_USER=onetl
-ONETL_ORA_PASSWORD=Yoequ2Hoeceit4ch
+ONETL_ORA_PASSWORD=123UsedForTestOnly@!
+ONETL_ORA_ROOT_USER=SYSTEM
+ONETL_ORA_ROOT_PASSWORD=123AdminUserForTests@!
 
 # FTP
 ONETL_FTP_HOST=ftp
 ONETL_FTP_PORT=21
 ONETL_FTP_USER=onetl
-ONETL_FTP_PASSWORD=awd123fd1
+ONETL_FTP_PASSWORD=123UsedForTestOnly@!
 
 # FTPS
 ONETL_FTPS_HOST=ftps
 ONETL_FTPS_PORT=21
 ONETL_FTPS_USER=onetl
-ONETL_FTPS_PASSWORD=awd123fd1
+ONETL_FTPS_PASSWORD=123UsedForTestOnly@!
 
 # S3
 ONETL_S3_HOST=s3
 ONETL_S3_PORT=9000
 ONETL_S3_ACCESS_KEY=onetl
-ONETL_S3_SECRET_KEY=woh3fogh3Biereu3quee1aidu9theiro
+ONETL_S3_SECRET_KEY=123UsedForTestOnly@!
 ONETL_S3_BUCKET=onetl
 
 # HDFS
@@ -85,7 +93,7 @@ ONETL_HDFS_IPC_PORT=9820
 ONETL_SFTP_HOST=sftp
 ONETL_SFTP_PORT=2222
 ONETL_SFTP_USER=onetl
-ONETL_SFTP_PASSWORD=AesujeifohgoaCu0Boosiet5aimeitho
+ONETL_SFTP_PASSWORD=123UsedForTestOnly@!
 
 # Samba
 ONETL_SAMBA_HOST=samba
@@ -95,10 +103,10 @@ ONETL_SAMBA_GID=1000
 ONETL_SAMBA_PORT=445
 ONETL_SAMBA_SHARE=SmbShare
 ONETL_SAMBA_USER=onetl
-ONETL_SAMBA_PASSWORD=awd123fd1
+ONETL_SAMBA_PASSWORD=123UsedForTestOnly@!
 
 # Webdav
 ONETL_WEBDAV_HOST=webdav
 ONETL_WEBDAV_PORT=80
 ONETL_WEBDAV_USER=onetl
-ONETL_WEBDAV_PASSWORD=awd123fd1
+ONETL_WEBDAV_PASSWORD=123UsedForTestOnly@!

.env.docker

@@ -1,79 +1,87 @@
 export TZ=UTC
+export SPARK_EXTERNAL_IP=$(docker network inspect onetl_onetl --format '{{ (index .IPAM.Config 0).Gateway }}')
 
 # Greenplum
 export ONETL_GP_HOST=localhost
 export ONETL_GP_PORT=5433
-export ONETL_GP_DATABASE=postgres
-export ONETL_GP_USER=gpadmin
-export ONETL_GP_PASSWORD=
+export ONETL_GP_DATABASE=onetl
+export ONETL_GP_USER=onetl
+export ONETL_GP_PASSWORD=123UsedForTestOnly@!
 
 # ClickHouse
 export ONETL_CH_HOST=localhost
 export ONETL_CH_PORT=8123
 export ONETL_CH_PORT_CLIENT=9001
 export ONETL_CH_DATABASE=default
 export ONETL_CH_USER=default
-export ONETL_CH_PASSWORD=
+export ONETL_CH_PASSWORD=123UsedForTestOnly@!
 
 # Kafka
 export ONETL_KAFKA_HOST=localhost
 export ONETL_KAFKA_PLAINTEXT_ANONYMOUS_PORT=9093
 export ONETL_KAFKA_PLAINTEXT_SASL_PORT=9095
 export ONETL_KAFKA_USER=onetl
-export ONETL_KAFKA_PASSWORD=uufoFae9sahSoidoo0eagaidaoreif6z
+export ONETL_KAFKA_PASSWORD=123UsedForTestOnly@!
 
 # Mongo
 export ONETL_MONGO_HOST=localhost
 export ONETL_MONGO_PORT=27017
-export ONETL_MONGO_DB=admin
+export ONETL_MONGO_DB=onetl
 export ONETL_MONGO_USER=onetl
-export ONETL_MONGO_PASSWORD=E4j7h!9A
+export ONETL_MONGO_PASSWORD=123UsedForTestOnly@!
+export ONETL_MONGO_ROOT_DB=admin
+export ONETL_MONGO_ROOT_USER=admin
+export ONETL_MONGO_ROOT_PASSWORD=123AdminUserForTests@!
 
 # MSSQL
 export ONETL_MSSQL_HOST=localhost
 export ONETL_MSSQL_PORT=1433
 export ONETL_MSSQL_DATABASE=onetl
 export ONETL_MSSQL_USER=onetl
-export ONETL_MSSQL_PASSWORD=7ellowEl7akey
+export ONETL_MSSQL_PASSWORD=123UsedForTestOnly@!
 
 # MySQL
 export ONETL_MYSQL_HOST=localhost
 export ONETL_MYSQL_PORT=3306
 export ONETL_MYSQL_DATABASE=onetl
 export ONETL_MYSQL_USER=onetl
-export ONETL_MYSQL_PASSWORD=ohbuz9Eochaj9saibooK3thooGa5aesh
+export ONETL_MYSQL_PASSWORD=123UsedForTestOnly@!
+export ONETL_MYSQL_ROOT_USER=root
+export ONETL_MYSQL_ROOT_PASSWORD=123RootUserForTestsOnly@!
 
 # Postgres
 export ONETL_PG_HOST=localhost
 export ONETL_PG_PORT=5432
 export ONETL_PG_DATABASE=onetl
 export ONETL_PG_USER=onetl
-export ONETL_PG_PASSWORD=ohtae0luxeshi1uraeluMoh9IShah7ai
+export ONETL_PG_PASSWORD=123UsedForTestOnly@!
 
 # Oracle
 export ONETL_ORA_HOST=localhost
 export ONETL_ORA_PORT=1522
 export ONETL_ORA_SERVICE_NAME=FREEPDB1
 export ONETL_ORA_USER=onetl
-export ONETL_ORA_PASSWORD=Yoequ2Hoeceit4ch
+export ONETL_ORA_PASSWORD=123UsedForTestOnly@!
+export ONETL_ORA_ROOT_USER=SYSTEM
+export ONETL_ORA_ROOT_PASSWORD=123AdminUserForTests@!
 
 # FTP
 export ONETL_FTP_HOST=localhost
 export ONETL_FTP_PORT=2121
 export ONETL_FTP_USER=onetl
-export ONETL_FTP_PASSWORD=awd123fd1
+export ONETL_FTP_PASSWORD=123UsedForTestOnly@!
 
 # FTPS
 export ONETL_FTPS_HOST=localhost
 export ONETL_FTPS_PORT=2122
 export ONETL_FTPS_USER=onetl
-export ONETL_FTPS_PASSWORD=awd123fd1
+export ONETL_FTPS_PASSWORD=123UsedForTestOnly@!
 
 # S3
 export ONETL_S3_HOST=localhost
 export ONETL_S3_PORT=9010
 export ONETL_S3_ACCESS_KEY=onetl
-export ONETL_S3_SECRET_KEY=woh3fogh3Biereu3quee1aidu9theiro
+export ONETL_S3_SECRET_KEY=123UsedForTestOnly@!
 export ONETL_S3_BUCKET=onetl
 
 # HDFS
@@ -85,7 +93,7 @@ export ONETL_HDFS_IPC_PORT=9820
 export ONETL_SFTP_HOST=localhost
 export ONETL_SFTP_PORT=2222
 export ONETL_SFTP_USER=onetl
-export ONETL_SFTP_PASSWORD=AesujeifohgoaCu0Boosiet5aimeitho
+export ONETL_SFTP_PASSWORD=123UsedForTestOnly@!
 
 # Samba
 export ONETL_SAMBA_HOST=localhost
@@ -95,10 +103,10 @@ export ONETL_SAMBA_GID=1000
 export ONETL_SAMBA_PORT=445
 export ONETL_SAMBA_SHARE=SmbShare
 export ONETL_SAMBA_USER=onetl
-export ONETL_SAMBA_PASSWORD=awd123fd1
+export ONETL_SAMBA_PASSWORD=123UsedForTestOnly@!
 
 # Webdav
 export ONETL_WEBDAV_HOST=localhost
 export ONETL_WEBDAV_PORT=8000
 export ONETL_WEBDAV_USER=onetl
-export ONETL_WEBDAV_PASSWORD=awd123fd1
+export ONETL_WEBDAV_PASSWORD=123UsedForTestOnly@!

.env.local

@@ -7,7 +7,7 @@ on:
       - master
 
 env:
-  DEFAULT_PYTHON: '3.12'
+  DEFAULT_PYTHON: '3.13'
 
 permissions:
   contents: read

.github/workflows/changelog.yml

@@ -14,9 +14,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  # flake8-commas is failing on Python 3.12
-  # as well as bandit https://github.com/PyCQA/bandit/issues/1077
-  DEFAULT_PYTHON: '3.11'
+  DEFAULT_PYTHON: '3.13'
 
 jobs:
   linters:
@@ -66,6 +64,8 @@ jobs:
             -r requirements/spark.txt \
             -r requirements/webdav.txt
 
+          pip install -U flake8-commas
+
           # Set the `CODEQL-PYTHON` environment variable to the Python executable
           # that includes the dependencies
           echo "CODEQL_PYTHON=$(which python)" >> $GITHUB_ENV

.github/workflows/codeql-analysis.yml

@@ -1,19 +1,19 @@
 min: &min
   # Clickhouse version with proper DateTime > DateTime64 comparison
   clickhouse-image: yandex/clickhouse-server
-  clickhouse-version: '21.1'
+  clickhouse-version: '21.1-alpine'
   spark-version: 2.3.1
   pydantic-version: 1
   python-version: '3.7'
   java-version: 8
-  os: ubuntu-latest
+  os: ubuntu-22.04
 
 max: &max
   clickhouse-image: clickhouse/clickhouse-server
-  clickhouse-version: 24.8.2.3-alpine
-  spark-version: 3.5.3
+  clickhouse-version: 25.1-alpine
+  spark-version: 3.5.4
   pydantic-version: 2
-  python-version: '3.12'
+  python-version: '3.13'
   java-version: 20
   os: ubuntu-latest
 
@@ -22,7 +22,7 @@ latest: &latest
   clickhouse-version: latest-alpine
   spark-version: latest
   pydantic-version: latest
-  python-version: '3.12'
+  python-version: '3.13'
   java-version: 20
   os: ubuntu-latest