Merge branch 'develop'

Author: dolfinus

.env.docker

@@ -87,6 +87,16 @@ ONETL_SFTP_PORT=2222
 ONETL_SFTP_USER=onetl
 ONETL_SFTP_PASSWORD=AesujeifohgoaCu0Boosiet5aimeitho
 
+# Samba
+ONETL_SAMBA_HOST=samba
+ONETL_SAMBA_PROTOCOL=SMB
+ONETL_SAMBA_UID=1000
+ONETL_SAMBA_GID=1000
+ONETL_SAMBA_PORT=445
+ONETL_SAMBA_SHARE=SmbShare
+ONETL_SAMBA_USER=onetl
+ONETL_SAMBA_PASSWORD=awd123fd1
+
 # Webdav
 ONETL_WEBDAV_HOST=webdav
 ONETL_WEBDAV_PORT=80

.env.local

@@ -87,6 +87,16 @@ export ONETL_SFTP_PORT=2222
 export ONETL_SFTP_USER=onetl
 export ONETL_SFTP_PASSWORD=AesujeifohgoaCu0Boosiet5aimeitho
 
+# Samba
+export ONETL_SAMBA_HOST=localhost
+export ONETL_SAMBA_PROTOCOL=SMB
+export ONETL_SAMBA_UID=1000
+export ONETL_SAMBA_GID=1000
+export ONETL_SAMBA_PORT=445
+export ONETL_SAMBA_SHARE=SmbShare
+export ONETL_SAMBA_USER=onetl
+export ONETL_SAMBA_PASSWORD=awd123fd1
+
 # Webdav
 export ONETL_WEBDAV_HOST=localhost
 export ONETL_WEBDAV_PORT=8000

.github/workflows/data/greenplum/matrix.yml

@@ -7,7 +7,7 @@ min: &min
 
 max: &max
   # Greenplum connector does not support Spark 3.3+
-  spark-version: 3.2.3
+  spark-version: 3.2.4
   python-version: '3.10'
   java-version: 11
   os: ubuntu-latest

.github/workflows/data/local-fs/matrix.yml

@@ -4,12 +4,18 @@ min: &min
   java-version: 8
   os: ubuntu-latest
 
-avro: &avro
+min_avro: &min_avro
   spark-version: 2.4.8
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
+min_excel: &min_excel
+  spark-version: 3.2.4
+  python-version: '3.7'
+  java-version: 8
+  os: ubuntu-latest
+
 max: &max
   spark-version: 3.4.1
   python-version: '3.11'
@@ -25,12 +31,15 @@ latest: &latest
 matrix:
   small:
   - <<: *max
-  - <<: *avro
+  - <<: *min_avro
+  - <<: *min_excel
   full:
   - <<: *min
-  - <<: *avro
+  - <<: *min_avro
+  - <<: *min_excel
   - <<: *max
   nightly:
   - <<: *min
-  - <<: *avro
+  - <<: *min_avro
+  - <<: *min_excel
   - <<: *latest

.github/workflows/data/mongodb/matrix.yml

@@ -1,6 +1,6 @@
 min: &min
   # MongoDB connector does not support Spark 2
-  spark-version: 3.2.3
+  spark-version: 3.2.4
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest

.github/workflows/data/s3/matrix.yml

@@ -2,7 +2,7 @@ min: &min
   # prior image versions returns empty content of bucket root, some kind of bug
   minio-version: 2021.3.17
   # Minimal Spark version with Hadoop 3.x support
-  spark-version: 3.2.3
+  spark-version: 3.2.4
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest

.github/workflows/data/samba/ignored.txt

@@ -0,0 +1 @@
+docs

.github/workflows/data/samba/matrix.yml

@@ -0,0 +1,18 @@
+min: &min
+  python-version: '3.7'
+  os: ubuntu-latest
+
+max: &max
+  python-version: '3.11'
+  os: ubuntu-latest
+
+matrix:
+  small:
+  - server-version: latest
+    <<: *max
+  full: &full
+  - server-version: latest
+    <<: *min
+  - server-version: latest
+    <<: *max
+  nightly: *full

.github/workflows/data/samba/tracked.txt

@@ -0,0 +1 @@
+**/samba*

.github/workflows/get-matrix.yml

@@ -41,6 +41,8 @@ on:
         value: ${{ jobs.get-matrix.outputs.matrix-s3 }}
       matrix-sftp:
         value: ${{ jobs.get-matrix.outputs.matrix-sftp }}
+      matrix-samba:
+        value: ${{ jobs.get-matrix.outputs.matrix-samba }}
       matrix-webdav:
         value: ${{ jobs.get-matrix.outputs.matrix-webdav }}
 
@@ -69,6 +71,7 @@ jobs:
       matrix-hdfs: ${{ toJson(fromJson(steps.matrix-hdfs.outputs.result)[steps.key-hdfs.outputs.key]) }}
       matrix-s3: ${{ toJson(fromJson(steps.matrix-s3.outputs.result)[steps.key-s3.outputs.key]) }}
       matrix-sftp: ${{ toJson(fromJson(steps.matrix-sftp.outputs.result)[steps.key-sftp.outputs.key]) }}
+      matrix-samba: ${{ toJson(fromJson(steps.matrix-samba.outputs.result)[steps.key-samba.outputs.key]) }}
       matrix-webdav: ${{ toJson(fromJson(steps.matrix-webdav.outputs.result)[steps.key-webdav.outputs.key]) }}
     steps:
     - name: Checkout code
@@ -635,6 +638,36 @@ jobs:
       with:
         cmd: yq -o=json '.matrix' .github/workflows/data/sftp/matrix.yml
 
+    - name: Check if Samba files are changed
+      id: changed-samba
+      uses: tj-actions/changed-files@v35
+      with:
+        files_from_source_file: .github/workflows/data/samba/tracked.txt
+        files_ignore_from_source_file: .github/workflows/data/samba/ignored.txt
+
+    - name: Print Samba files changed
+      run: |
+        echo '${{ steps.changed-samba.outputs.all_changed_files }}'
+
+    - name: Calculate Samba matrix key
+      id: key-samba
+      run: |
+        if ${{ inputs.nightly }}; then
+          key=nightly
+        elif ${{ steps.changed-base.outputs.any_changed }} || ${{ steps.changed-file.outputs.any_changed }} || ${{ steps.changed-samba.outputs.any_changed }}; then
+          key=full
+        else
+          key=small
+        fi
+        echo key=$key
+        echo key=$key >> $GITHUB_OUTPUT
+
+    - name: Get Samba matrix
+      id: matrix-samba
+      uses: mikefarah/yq@v4.33.3
+      with:
+        cmd: yq -o=json '.matrix' .github/workflows/data/samba/matrix.yml
+
     - name: Check if WebDAV files are changed
       id: changed-webdav
       uses: tj-actions/changed-files@v35