Merge branch 'main' into feat/s3-region-handling

Author: Techassi

.github/workflows/build.yml

@@ -26,7 +26,8 @@ env:
   CARGO_TERM_COLOR: always
   CARGO_INCREMENTAL: '0'
   CARGO_PROFILE_DEV_DEBUG: '0'
-  RUST_TOOLCHAIN_VERSION: "1.82.0"
+  RUST_TOOLCHAIN_VERSION: "1.84.1"
+  PYTHON_VERSION: "3.12"
   RUSTFLAGS: "-D warnings"
   RUSTDOCFLAGS: "-D warnings"
   RUST_LOG: "info"
@@ -49,8 +50,9 @@ jobs:
           version: ubuntu-latest
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
+          persist-credentials: false
           submodules: recursive
-      - uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203
+      - uses: dtolnay/rust-toolchain@c5a29ddb4d9d194e7c84ec8c3fba61b1c31fee8c
         with:
           toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }}
       - uses: Swatinem/rust-cache@f0deed1e0edfc6a9be95417288c0e1099b1eeec3 # v2.7.7
@@ -118,8 +120,9 @@ jobs:
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
+          persist-credentials: false
           submodules: recursive
-      - uses: EmbarkStudios/cargo-deny-action@8371184bd11e21dcf8ac82ebf8c9c9f74ebf7268 # v2.0.1
+      - uses: EmbarkStudios/cargo-deny-action@8d73959fce1cdc8989f23fdf03bec6ae6a6576ef # v2.0.7
         with:
           command: check ${{ matrix.checks }}
 
@@ -129,8 +132,9 @@ jobs:
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
+          persist-credentials: false
           submodules: recursive
-      - uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203
+      - uses: dtolnay/rust-toolchain@c5a29ddb4d9d194e7c84ec8c3fba61b1c31fee8c
         with:
           toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }}
           components: rustfmt
@@ -147,8 +151,9 @@ jobs:
           version: ubuntu-latest
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
+          persist-credentials: false
           submodules: recursive
-      - uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203
+      - uses: dtolnay/rust-toolchain@c5a29ddb4d9d194e7c84ec8c3fba61b1c31fee8c
         with:
           toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }}
           components: clippy
@@ -183,7 +188,7 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           submodules: recursive
-      - uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203
+      - uses: dtolnay/rust-toolchain@c5a29ddb4d9d194e7c84ec8c3fba61b1c31fee8c
         with:
           toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }}
           components: rustfmt
@@ -204,8 +209,9 @@ jobs:
           version: ubuntu-latest
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
+          persist-credentials: false
           submodules: recursive
-      - uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203
+      - uses: dtolnay/rust-toolchain@c5a29ddb4d9d194e7c84ec8c3fba61b1c31fee8c
         with:
           toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }}
       - uses: Swatinem/rust-cache@f0deed1e0edfc6a9be95417288c0e1099b1eeec3 # v2.7.7
@@ -224,10 +230,11 @@ jobs:
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
+          persist-credentials: false
           submodules: recursive
       - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
-          python-version: '3.12'
+          python-version: ${{ env.PYTHON_VERSION }}
       - name: Install jinja2-cli
         run: pip install jinja2-cli==0.8.2
       - name: Regenerate charts
@@ -262,13 +269,14 @@ jobs:
           version: ubuntu-latest
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
+          persist-credentials: false
           submodules: recursive
       - name: Set up Helm
         uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0
         with:
           version: v3.16.1
       - name: Set up cargo
-        uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203
+        uses: dtolnay/rust-toolchain@c5a29ddb4d9d194e7c84ec8c3fba61b1c31fee8c
         with:
           toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }}
       - uses: Swatinem/rust-cache@f0deed1e0edfc6a9be95417288c0e1099b1eeec3 # v2.7.7
@@ -332,9 +340,10 @@ jobs:
           version: ${{ matrix.runner }}
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
+          persist-credentials: false
           submodules: recursive
-      - uses: cachix/install-nix-action@ba0dd844c9180cbf77aa72a116d6fbc515d0e87b # v27
-      - uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203
+      - uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30
+      - uses: dtolnay/rust-toolchain@c5a29ddb4d9d194e7c84ec8c3fba61b1c31fee8c
         with:
           toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }}
           components: rustfmt
@@ -371,9 +380,9 @@ jobs:
       # default value in the makefile if called from this action, but not otherwise (i.e. when called locally).
       # This is needed for the HELM_REPO variable.
       - name: Install cosign
-        uses: sigstore/cosign-installer@4959ce089c160fddf62f7b42464195ba1a56d382 # v3.6.0
+        uses: sigstore/cosign-installer@d7d6bc7722e3daa8354c50bcb52f4837da5e9b6a # v3.8.1
       - name: Install syft
-        uses: anchore/sbom-action/download-syft@61119d458adab75f756bc0b9e4bde25725f86a7a # v0.17.2
+        uses: anchore/sbom-action/download-syft@f325610c9f50a54015d37c8d16cb3b0e2c8f4de0 # v0.18.0
       - name: Build Docker image and Helm chart
         run: |
           # Installing helm and yq on ubicloud-standard-8-arm only
@@ -417,10 +426,11 @@ jobs:
       OCI_REGISTRY_SDP_CHARTS_USERNAME: "robot$sdp-charts+github-action-build"
     steps:
       - name: Install cosign
-        uses: sigstore/cosign-installer@4959ce089c160fddf62f7b42464195ba1a56d382 # v3.6.0
+        uses: sigstore/cosign-installer@d7d6bc7722e3daa8354c50bcb52f4837da5e9b6a # v3.8.1
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
+          persist-credentials: false
           submodules: recursive
       # This step checks if the current run was triggered by a push to a pr (or a pr being created).
       # If this is the case it changes the version of this project in all Cargo.toml files to include the suffix

.github/workflows/general_daily_security.yml

@@ -10,11 +10,15 @@ on:
     - cron: '15 4 * * *'
   workflow_dispatch:
 
+permissions: {}
+
 jobs:
   audit:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          persist-credentials: false
       - uses: rustsec/audit-check@dd51754d4e59da7395a4cd9b593f0ff2d61a9b95 # v1.4.1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/pr_pre-commit.yaml

@@ -21,9 +21,10 @@ jobs:
           version: ubuntu-latest
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
-          fetch-depth: 0
+          persist-credentials: false
           submodules: recursive
-      - uses: stackabletech/actions/run-pre-commit@5b66858af3597c4ea34f9b33664b8034a1d28427 # v0.3.0
+          fetch-depth: 0
+      - uses: stackabletech/actions/run-pre-commit@2d3d7ddad981ae09901d45a0f6bf30c2658b1b78 # v0.7.0
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           rust: ${{ env.RUST_TOOLCHAIN_VERSION }}

.pre-commit-config.yaml

@@ -14,19 +14,13 @@ repos:
         args: ["--allow-missing-credentials"]
       - id: detect-private-key
 
-  - repo: https://github.com/doublify/pre-commit-rust
-    rev: eeee35a89e69d5772bdee97db1a6a898467b686e # 1.0
-    hooks:
-      - id: clippy
-        args: ["--all-targets", "--", "-D", "warnings"]
-
   - repo: https://github.com/adrienverge/yamllint
     rev: 81e9f98ffd059efe8aa9c1b1a42e5cce61b640c6 # 1.35.1
     hooks:
       - id: yamllint
 
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: aa975a18c9a869648007d33864034dbc7481fe5e # 0.42.0
+    rev: 586c3ea3f51230da42bab657c6a32e9e66c364f0 # 0.44.0
     hooks:
       - id: markdownlint
         types: [text]
@@ -42,15 +36,15 @@ repos:
   # If you do not, you will need to delete the cached ruff binary shown in the
   # error message
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 8983acb92ee4b01924893632cf90af926fa608f0 # 0.7.0
+    rev: 2c8dce6094fa2b4b668e74f694ca63ceffd38614 # 0.9.9
     hooks:
       # Run the linter.
       - id: ruff
       # Run the formatter.
       - id: ruff-format
 
   - repo: https://github.com/rhysd/actionlint
-    rev: 4e683ab8014a63fafa117492a0c6053758e6d593 # 1.7.3
+    rev: 03d0035246f3e81f36aed592ffb4bebf33a03106 # 1.7.7
     hooks:
       - id: actionlint
 
@@ -74,10 +68,20 @@ repos:
         entry: cargo test
         stages: [pre-commit, pre-merge-commit, manual]
         pass_filenames: false
+        files: \.rs$|Cargo\.(toml|lock)
 
       - id: cargo-rustfmt
         name: cargo-rustfmt
         language: system
         entry: cargo +nightly-2025-01-15 fmt --all -- --check
         stages: [pre-commit]
         pass_filenames: false
+        files: \.rs$
+
+      - id: cargo-clippy
+        name: cargo-clippy
+        language: system
+        entry: cargo clippy --all-targets -- -D warnings
+        stages: [pre-commit]
+        pass_filenames: false
+        files: \.rs$

docs/modules/hive/pages/usage-guide/data-storage.adoc

@@ -1,11 +1,18 @@
 = Data storage backends
 :description: Hive supports metadata storage on S3 and HDFS. Configure S3 with S3Connection and HDFS with configMap in clusterConfig.
 
-Hive does not store data, only metadata. It can store metadata about data stored in various places. The Stackable Operator currently supports S3 and HFS.
+You can operate the Hive metastore service (HMS) without S3 or HDFS.
+Its whole purpose is to store metadata such as "Table foo has columns a, b and c and is stored as parquet in local://tmp/hive/foo".
 
-== [[s3]]S3 support
+However, as soon as you start storing metadata in the HMS that refers to a `s3a://` or `hdfs://` locations, HMS will actually do some operations on the filesystem. This can be e.g. checking if the table location exists, creating it in case it is missing.
 
-Hive supports creating tables in S3 compatible object stores.
+So if you are storing tables in S3 (or HDFS for that matter), you need to give the HMS access to that filesystem as well.
+The Stackable Operator currently supports S3 and HFS.
+
+[s3]
+== S3 support
+
+HMS supports creating tables in S3 compatible object stores.
 To use this feature you need to provide connection details for the object store using the xref:concepts:s3.adoc[S3Connection] in the top level `clusterConfig`.
 
 An example usage can look like this:
@@ -22,10 +29,10 @@ clusterConfig:
         secretClass: simple-hive-s3-secret-class
 ----
 
+[hdfs]
+== Apache HDFS support
 
-== [[hdfs]]Apache HDFS support
-
-As well as S3, Hive also supports creating tables in HDFS.
+As well as S3, HMS also supports creating tables in HDFS.
 You can add the HDFS connection in the top level `clusterConfig` as follows:
 
 [source,yaml]

rust-toolchain.toml

@@ -1,3 +1,3 @@
 # DO NOT EDIT, this file is generated by operator-templating
 [toolchain]
-channel = "1.82.0"
+channel = "1.84.1"