diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b3628cd..ef8d45b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ All notable changes to this project will be documented in this file. - Run a `containerdebug` process in the background of each Hive container to collect debugging information ([#554]). - Aggregate emitted Kubernetes events on the CustomResources ([#560]). - Support configuring JVM arguments ([#572]). +- Support for S3 region ([#574]). ### Changed @@ -27,6 +28,7 @@ All notable changes to this project will be documented in this file. [#560]: https://github.com/stackabletech/hive-operator/pull/560 [#561]: https://github.com/stackabletech/hive-operator/pull/561 [#572]: https://github.com/stackabletech/hive-operator/pull/572 +[#574]: https://github.com/stackabletech/hive-operator/pull/574 ## [24.11.1] - 2025-01-10 diff --git a/Cargo.lock b/Cargo.lock index 384ecd66..93acb084 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2126,15 +2126,14 @@ checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "ring" -version = "0.17.8" +version = "0.17.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +checksum = "70ac5d832aa16abd7d1def883a8545280c20a60f523a370aa3a9617c2b8550ee" dependencies = [ "cc", "cfg-if", "getrandom", "libc", - "spin", "untrusted", "windows-sys 0.52.0", ] @@ -2532,12 +2531,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "spin" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" - [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -2572,8 +2565,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.85.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#59506c6202778889a27b6ae8153457e60a49c68d" +version = "0.87.2" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.87.2#bc176bfc23f15533cdb3b7a7e7a773d4f29891e1" dependencies = [ "chrono", "clap", @@ -2611,7 +2604,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#59506c6202778889a27b6ae8153457e60a49c68d" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.87.2#bc176bfc23f15533cdb3b7a7e7a773d4f29891e1" dependencies = [ "darling", "proc-macro2", @@ -2622,7 +2615,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.0.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#59506c6202778889a27b6ae8153457e60a49c68d" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.87.2#bc176bfc23f15533cdb3b7a7e7a773d4f29891e1" dependencies = [ "kube 0.98.0", "semver", @@ -2663,18 +2656,18 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "strum" -version = "0.26.3" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +checksum = "f64def088c51c9510a8579e3c5d67c65349dcf755e5479ad3d010aa6454e2c32" dependencies = [ "strum_macros", ] [[package]] name = "strum_macros" -version = "0.26.4" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8" dependencies = [ "heck", "proc-macro2", diff --git a/Cargo.nix b/Cargo.nix index 951c282b..6cb22a08 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -1594,7 +1594,7 @@ rec { "default" = [ "Debug" "Clone" "Copy" "PartialEq" "Eq" "PartialOrd" "Ord" "Hash" "Default" "Deref" "DerefMut" "Into" ]; "full" = [ "syn/full" ]; }; - resolvedDefaultFeatures = [ "Clone" "Debug" "Default" "Hash" "PartialEq" ]; + resolvedDefaultFeatures = [ "Clone" "Debug" "Default" "Eq" "Hash" "PartialEq" ]; }; "either" = rec { crateName = "either"; @@ -6841,13 +6841,10 @@ rec { }; "ring" = rec { crateName = "ring"; - version = "0.17.8"; + version = "0.17.13"; edition = "2021"; - links = "ring_core_0_17_8"; - sha256 = "03fwlb1ssrmfxdckvqv033pfmk01rhx9ynwi7r186dcfcp5s8zy1"; - authors = [ - "Brian Smith " - ]; + links = "ring_core_0_17_13_"; + sha256 = "1vjhhlmpqqd9lc53ffjj1yk203188n2km27g3myvssm15a1mvb3h"; dependencies = [ { name = "cfg-if"; @@ -6862,14 +6859,13 @@ rec { name = "libc"; packageId = "libc"; usesDefaultFeatures = false; - target = { target, features }: ((("android" == target."os" or null) || ("linux" == target."os" or null)) && (("aarch64" == target."arch" or null) || ("arm" == target."arch" or null))); + target = { target, features }: ((("aarch64" == target."arch" or null) && ("little" == target."endian" or null)) && ("apple" == target."vendor" or null) && (("ios" == target."os" or null) || ("macos" == target."os" or null) || ("tvos" == target."os" or null) || ("visionos" == target."os" or null) || ("watchos" == target."os" or null))); } { - name = "spin"; - packageId = "spin"; + name = "libc"; + packageId = "libc"; usesDefaultFeatures = false; - target = { target, features }: (("aarch64" == target."arch" or null) || ("arm" == target."arch" or null) || ("x86" == target."arch" or null) || ("x86_64" == target."arch" or null)); - features = [ "once" ]; + target = { target, features }: (((("aarch64" == target."arch" or null) && ("little" == target."endian" or null)) || (("arm" == target."arch" or null) && ("little" == target."endian" or null))) && (("android" == target."os" or null) || ("linux" == target."os" or null))); } { name = "untrusted"; @@ -6878,7 +6874,7 @@ rec { { name = "windows-sys"; packageId = "windows-sys 0.52.0"; - target = { target, features }: (("aarch64" == target."arch" or null) && ("windows" == target."os" or null)); + target = { target, features }: ((("aarch64" == target."arch" or null) && ("little" == target."endian" or null)) && ("windows" == target."os" or null)); features = [ "Win32_Foundation" "Win32_System_Threading" ]; } ]; @@ -8061,31 +8057,6 @@ rec { }; resolvedDefaultFeatures = [ "all" ]; }; - "spin" = rec { - crateName = "spin"; - version = "0.9.8"; - edition = "2015"; - sha256 = "0rvam5r0p3a6qhc18scqpvpgb3ckzyqxpgdfyjnghh8ja7byi039"; - authors = [ - "Mathijs van de Nes " - "John Ericson " - "Joshua Barretto " - ]; - features = { - "barrier" = [ "mutex" ]; - "default" = [ "lock_api" "mutex" "spin_mutex" "rwlock" "once" "lazy" "barrier" ]; - "fair_mutex" = [ "mutex" ]; - "lazy" = [ "once" ]; - "lock_api" = [ "lock_api_crate" ]; - "lock_api_crate" = [ "dep:lock_api_crate" ]; - "portable-atomic" = [ "dep:portable-atomic" ]; - "portable_atomic" = [ "portable-atomic" ]; - "spin_mutex" = [ "mutex" ]; - "ticket_mutex" = [ "mutex" ]; - "use_ticket_mutex" = [ "mutex" "ticket_mutex" ]; - }; - resolvedDefaultFeatures = [ "once" ]; - }; "stable_deref_trait" = rec { crateName = "stable_deref_trait"; version = "1.2.0"; @@ -8211,13 +8182,13 @@ rec { }; "stackable-operator" = rec { crateName = "stackable-operator"; - version = "0.85.0"; + version = "0.87.2"; edition = "2021"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "59506c6202778889a27b6ae8153457e60a49c68d"; - sha256 = "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5"; + rev = "bc176bfc23f15533cdb3b7a7e7a773d4f29891e1"; + sha256 = "0cqz1xmj3vbm5hm9x6wbgg2l265s30j5j5609wmg68p6giywh82a"; }; libName = "stackable_operator"; authors = [ @@ -8250,7 +8221,7 @@ rec { name = "educe"; packageId = "educe"; usesDefaultFeatures = false; - features = [ "Clone" "Debug" "Default" "PartialEq" ]; + features = [ "Clone" "Debug" "Default" "PartialEq" "Eq" ]; } { name = "either"; @@ -8376,8 +8347,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "59506c6202778889a27b6ae8153457e60a49c68d"; - sha256 = "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5"; + rev = "bc176bfc23f15533cdb3b7a7e7a773d4f29891e1"; + sha256 = "0cqz1xmj3vbm5hm9x6wbgg2l265s30j5j5609wmg68p6giywh82a"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -8411,8 +8382,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "59506c6202778889a27b6ae8153457e60a49c68d"; - sha256 = "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5"; + rev = "bc176bfc23f15533cdb3b7a7e7a773d4f29891e1"; + sha256 = "0cqz1xmj3vbm5hm9x6wbgg2l265s30j5j5609wmg68p6giywh82a"; }; libName = "stackable_shared"; authors = [ @@ -8558,9 +8529,9 @@ rec { }; "strum" = rec { crateName = "strum"; - version = "0.26.3"; - edition = "2018"; - sha256 = "01lgl6jvrf4j28v5kmx9bp480ygf1nhvac8b4p7rcj9hxw50zv4g"; + version = "0.27.1"; + edition = "2021"; + sha256 = "0cic9r2sc2h17nnpjm2yfp7rsd35gkbcbqvrhl553jaiih4fykgn"; authors = [ "Peter Glotfelty " ]; @@ -8571,12 +8542,6 @@ rec { optional = true; } ]; - devDependencies = [ - { - name = "strum_macros"; - packageId = "strum_macros"; - } - ]; features = { "default" = [ "std" ]; "derive" = [ "strum_macros" ]; @@ -8587,9 +8552,9 @@ rec { }; "strum_macros" = rec { crateName = "strum_macros"; - version = "0.26.4"; - edition = "2018"; - sha256 = "1gl1wmq24b8md527cpyd5bw9rkbqldd7k1h38kf5ajd2ln2ywssc"; + version = "0.27.1"; + edition = "2021"; + sha256 = "1s7x07nkrgjfvxrvcdjw6qanad4c55yjnd32bph9q3xgpid8qyn7"; procMacro = true; authors = [ "Peter Glotfelty " @@ -8614,7 +8579,7 @@ rec { { name = "syn"; packageId = "syn 2.0.96"; - features = [ "parsing" "extra-traits" ]; + features = [ "parsing" ]; } ]; diff --git a/Cargo.toml b/Cargo.toml index 79f84b5a..556e601d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ repository = "https://github.com/stackabletech/hive-operator" [workspace.dependencies] stackable-versioned = { git = "https://github.com/stackabletech/operator-rs.git", features = ["k8s"], tag = "stackable-versioned-0.5.0" } -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.85.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.87.2" } product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.7.0" } anyhow = "1.0" @@ -28,7 +28,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_yaml = "0.9" snafu = "0.8" -strum = { version = "0.26", features = ["derive"] } +strum = { version = "0.27", features = ["derive"] } tokio = { version = "1.40", features = ["full"] } tracing = "0.1" diff --git a/crate-hashes.json b/crate-hashes.json index c7d32c3a..e02a942d 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,7 +1,7 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#stackable-operator-derive@0.3.1": "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#stackable-operator@0.85.0": "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#stackable-shared@0.0.1": "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.87.2#stackable-operator-derive@0.3.1": "0cqz1xmj3vbm5hm9x6wbgg2l265s30j5j5609wmg68p6giywh82a", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.87.2#stackable-operator@0.87.2": "0cqz1xmj3vbm5hm9x6wbgg2l265s30j5j5609wmg68p6giywh82a", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.87.2#stackable-shared@0.0.1": "0cqz1xmj3vbm5hm9x6wbgg2l265s30j5j5609wmg68p6giywh82a", "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-versioned-0.5.0#k8s-version@0.1.2": "1x2pfibrsysmkkmajyj30qkwsjf3rzmc3dxsd09jb9r4x7va6mr6", "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-versioned-0.5.0#stackable-versioned-macros@0.5.0": "1x2pfibrsysmkkmajyj30qkwsjf3rzmc3dxsd09jb9r4x7va6mr6", "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-versioned-0.5.0#stackable-versioned@0.5.0": "1x2pfibrsysmkkmajyj30qkwsjf3rzmc3dxsd09jb9r4x7va6mr6", diff --git a/deploy/helm/hive-operator/crds/crds.yaml b/deploy/helm/hive-operator/crds/crds.yaml index 2cc67500..f0ca0792 100644 --- a/deploy/helm/hive-operator/crds/crds.yaml +++ b/deploy/helm/hive-operator/crds/crds.yaml @@ -157,6 +157,20 @@ spec: minimum: 0.0 nullable: true type: integer + region: + default: + name: us-east-1 + description: |- + Bucket region used for signing headers (sigv4). + + This defaults to `us-east-1` which is compatible with other implementations such as Minio. + + WARNING: Some products use the Hadoop S3 implementation which falls back to us-east-2. + properties: + name: + default: us-east-1 + type: string + type: object tls: description: Use a TLS connection. If not specified no TLS will be used. nullable: true diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 38706116..0b371df5 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -625,6 +625,11 @@ fn build_metastore_rolegroup_config_map( Some(s3.endpoint().context(ConfigureS3Snafu)?.to_string()), ); + data.insert( + MetaStoreConfig::S3_REGION_NAME.to_string(), + Some(s3.region.name.clone()), + ); + if let Some((access_key_file, secret_key_file)) = s3.credentials_mount_paths() { // Will be replaced by config-utils data.insert( diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index ac5d6684..655a3b9c 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -460,6 +460,7 @@ impl MetaStoreConfig { pub const METASTORE_WAREHOUSE_DIR: &'static str = "hive.metastore.warehouse.dir"; // S3 pub const S3_ENDPOINT: &'static str = "fs.s3a.endpoint"; + pub const S3_REGION_NAME: &'static str = "fs.s3a.endpoint.region"; pub const S3_ACCESS_KEY: &'static str = "fs.s3a.access.key"; pub const S3_SECRET_KEY: &'static str = "fs.s3a.secret.key"; pub const S3_SSL_ENABLED: &'static str = "fs.s3a.connection.ssl.enabled";