From 10d869fe962eca0d2bcaf8f2c4caa7fb67e3b54c Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 31 Jan 2025 11:17:22 +0100 Subject: [PATCH 01/10] chore: Remove separate CRD crate --- Cargo.lock | 19 ------- Cargo.toml | 7 +-- rust/crd/Cargo.toml | 25 --------- rust/operator-binary/Cargo.toml | 2 - .../src/authentication/ldap.rs | 11 ++-- .../operator-binary/src/authentication/mod.rs | 16 +++--- .../src/authentication/oidc.rs | 4 +- rust/operator-binary/src/config.rs | 5 +- .../src/crd}/affinity.rs | 12 ++--- .../src/crd}/authentication.rs | 13 ++--- .../src/crd}/authorization.rs | 0 .../src => operator-binary/src/crd}/memory.rs | 5 +- .../lib.rs => operator-binary/src/crd/mod.rs} | 39 +++++++------- .../src/crd}/resource.rs | 32 +++++------ .../src/crd}/security.rs | 12 ++--- .../src/crd}/storage.rs | 0 .../src => operator-binary/src/crd}/tls.rs | 3 +- rust/operator-binary/src/discovery.rs | 10 ++-- rust/operator-binary/src/druid_controller.rs | 54 +++++++++---------- rust/operator-binary/src/extensions.rs | 10 ++-- rust/operator-binary/src/internal_secret.rs | 16 +++--- rust/operator-binary/src/main.rs | 27 +++++----- .../src/operations/graceful_shutdown.rs | 6 +-- rust/operator-binary/src/operations/pdb.rs | 6 ++- rust/operator-binary/src/product_logging.rs | 9 ++-- .../crd}/resource_merge/druid_cluster.yaml | 0 .../crd}/resource_merge/segment_cache.yaml | 0 .../crd}/role_service/druid_cluster.yaml | 0 rustfmt.toml | 2 + 29 files changed, 152 insertions(+), 193 deletions(-) delete mode 100644 rust/crd/Cargo.toml rename rust/{crd/src => operator-binary/src/crd}/affinity.rs (98%) rename rust/{crd/src => operator-binary/src/crd}/authentication.rs (99%) rename rust/{crd/src => operator-binary/src/crd}/authorization.rs (100%) rename rust/{crd/src => operator-binary/src/crd}/memory.rs (99%) rename rust/{crd/src/lib.rs => operator-binary/src/crd/mod.rs} (99%) rename rust/{crd/src => operator-binary/src/crd}/resource.rs (97%) rename rust/{crd/src => operator-binary/src/crd}/security.rs (99%) rename rust/{crd/src => operator-binary/src/crd}/storage.rs (100%) rename rust/{crd/src => operator-binary/src/crd}/tls.rs (98%) rename rust/{crd/test/resources => operator-binary/test/resources/crd}/resource_merge/druid_cluster.yaml (100%) rename rust/{crd/test/resources => operator-binary/test/resources/crd}/resource_merge/segment_cache.yaml (100%) rename rust/{crd/test/resources => operator-binary/test/resources/crd}/role_service/druid_cluster.yaml (100%) create mode 100644 rustfmt.toml diff --git a/Cargo.lock b/Cargo.lock index 6a7e97f3..35277d06 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2382,24 +2382,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" -[[package]] -name = "stackable-druid-crd" -version = "0.0.0-dev" -dependencies = [ - "indoc", - "product-config", - "rstest", - "semver", - "serde", - "serde_json", - "serde_yaml", - "snafu 0.8.5", - "stackable-operator", - "strum", - "tokio", - "tracing", -] - [[package]] name = "stackable-druid-operator" version = "0.0.0-dev" @@ -2420,7 +2402,6 @@ dependencies = [ "serde_json", "serde_yaml", "snafu 0.8.5", - "stackable-druid-crd", "stackable-operator", "strum", "tokio", diff --git a/Cargo.toml b/Cargo.toml index b95d91a0..d48513df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["rust/crd", "rust/operator-binary"] +members = ["rust/operator-binary"] resolver = "2" [workspace.package] @@ -10,6 +10,9 @@ edition = "2021" repository = "https://github.com/stackabletech/druid-operator" [workspace.dependencies] +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.85.0" } +product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.7.0" } + anyhow = "1.0" built = { version = "0.7", features = ["chrono", "git2"] } clap = "4.5" @@ -18,7 +21,6 @@ fnv = "1.0" futures = { version = "0.3", features = ["compat"] } indoc = "2.0" openssl = "0.10" -product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.7.0" } pin-project = "1.1" rstest = "0.24" semver = "1.0" @@ -26,7 +28,6 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_yaml = "0.9" snafu = "0.8" -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.85.0" } strum = { version = "0.26", features = ["derive"] } tokio = { version = "1.40", features = ["full"] } tracing = "0.1" diff --git a/rust/crd/Cargo.toml b/rust/crd/Cargo.toml deleted file mode 100644 index 2e2aa3aa..00000000 --- a/rust/crd/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -name = "stackable-druid-crd" -description = "Contains the Apache Druid CRD structs and utilities" -version.workspace = true -authors.workspace = true -license.workspace = true -edition.workspace = true -repository.workspace = true -publish = false - -[dependencies] -indoc.workspace = true -product-config.workspace = true -semver.workspace = true -serde.workspace = true -serde_json.workspace = true -stackable-operator.workspace = true -strum.workspace = true -tracing.workspace = true -snafu.workspace = true - -[dev-dependencies] -rstest.workspace = true -serde_yaml.workspace = true -tokio.workspace = true diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index e282d4ca..5eaa4564 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -9,8 +9,6 @@ repository.workspace = true publish = false [dependencies] -stackable-druid-crd = { path = "../crd" } - anyhow.workspace = true clap.workspace = true const_format.workspace = true diff --git a/rust/operator-binary/src/authentication/ldap.rs b/rust/operator-binary/src/authentication/ldap.rs index ac8e37d9..bfed3c82 100644 --- a/rust/operator-binary/src/authentication/ldap.rs +++ b/rust/operator-binary/src/authentication/ldap.rs @@ -6,12 +6,11 @@ use stackable_operator::{ commons::authentication::ldap::AuthenticationProvider, }; -use stackable_druid_crd::security::{ - add_cert_to_trust_store_cmd, STACKABLE_TLS_DIR, TLS_STORE_PASSWORD, -}; - -use crate::authentication::{ - AddLdapVolumesSnafu, ConstructLdapEndpointUrlSnafu, Error, MissingLdapBindCredentialsSnafu, +use crate::{ + authentication::{ + AddLdapVolumesSnafu, ConstructLdapEndpointUrlSnafu, Error, MissingLdapBindCredentialsSnafu, + }, + crd::security::{add_cert_to_trust_store_cmd, STACKABLE_TLS_DIR, TLS_STORE_PASSWORD}, }; fn add_authenticator_config( diff --git a/rust/operator-binary/src/authentication/mod.rs b/rust/operator-binary/src/authentication/mod.rs index 75c67d07..8a92b69a 100644 --- a/rust/operator-binary/src/authentication/mod.rs +++ b/rust/operator-binary/src/authentication/mod.rs @@ -1,11 +1,6 @@ use std::collections::BTreeMap; use snafu::Snafu; -use stackable_druid_crd::{ - authentication::{AuthenticationClassResolved, AuthenticationClassesResolved}, - security::{ESCALATOR_INTERNAL_CLIENT_PASSWORD_ENV, INTERNAL_INITIAL_CLIENT_PASSWORD_ENV}, - DruidCluster, DruidRole, -}; use stackable_operator::{ builder::pod::{container::ContainerBuilder, PodBuilder}, commons::{ @@ -20,11 +15,18 @@ use stackable_operator::{ k8s_openapi::api::core::v1::EnvVar, }; +use crate::{ + crd::{ + authentication::{AuthenticationClassResolved, AuthenticationClassesResolved}, + security::{ESCALATOR_INTERNAL_CLIENT_PASSWORD_ENV, INTERNAL_INITIAL_CLIENT_PASSWORD_ENV}, + DruidCluster, DruidRole, + }, + internal_secret::{build_shared_internal_secret_name, env_var_from_secret}, +}; + pub mod ldap; pub mod oidc; -use crate::internal_secret::{build_shared_internal_secret_name, env_var_from_secret}; - type Result = std::result::Result; #[derive(Snafu, Debug)] diff --git a/rust/operator-binary/src/authentication/oidc.rs b/rust/operator-binary/src/authentication/oidc.rs index 8fb65270..13ee72e4 100644 --- a/rust/operator-binary/src/authentication/oidc.rs +++ b/rust/operator-binary/src/authentication/oidc.rs @@ -1,9 +1,6 @@ use std::collections::BTreeMap; use snafu::ResultExt; -use stackable_druid_crd::{ - security::add_cert_to_jvm_trust_store_cmd, DruidRole, COOKIE_PASSPHRASE_ENV, -}; use stackable_operator::{ builder::pod::{container::ContainerBuilder, PodBuilder}, commons::authentication::oidc::{AuthenticationProvider, ClientAuthenticationOptions}, @@ -12,6 +9,7 @@ use stackable_operator::{ use crate::{ authentication::{AddOidcVolumesSnafu, ConstructOidcWellKnownUrlSnafu, Error}, + crd::{security::add_cert_to_jvm_trust_store_cmd, DruidRole, COOKIE_PASSPHRASE_ENV}, internal_secret::env_var_from_secret, }; diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs index 908718e1..8f3af715 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config.rs @@ -1,10 +1,11 @@ use indoc::formatdoc; use snafu::{ResultExt, Snafu}; -use stackable_druid_crd::{ +use stackable_operator::memory::MemoryQuantity; + +use crate::crd::{ DruidRole, JVM_SECURITY_PROPERTIES_FILE, LOG4J2_CONFIG, RW_CONFIG_DIRECTORY, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD, }; -use stackable_operator::memory::MemoryQuantity; #[derive(Snafu, Debug)] pub enum Error { diff --git a/rust/crd/src/affinity.rs b/rust/operator-binary/src/crd/affinity.rs similarity index 98% rename from rust/crd/src/affinity.rs rename to rust/operator-binary/src/crd/affinity.rs index 275d10bd..6f0b3b9c 100644 --- a/rust/crd/src/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -3,7 +3,7 @@ use stackable_operator::{ k8s_openapi::api::core::v1::{PodAffinity, PodAntiAffinity}, }; -use crate::{DeepStorageSpec, DruidRole, HdfsDeepStorageSpec, APP_NAME}; +use crate::crd::{DeepStorageSpec, DruidRole, HdfsDeepStorageSpec, APP_NAME}; /// Please have a look at the architecture diagram in /// to understand which roles do communicate with each other. @@ -32,7 +32,7 @@ pub fn get_affinity( DruidRole::Historical | // Ingests data DruidRole::MiddleManager => { - if let DeepStorageSpec::HDFS(HdfsDeepStorageSpec { + if let DeepStorageSpec::Hdfs(HdfsDeepStorageSpec { config_map_name: hdfs_discovery_cm_name, .. }) = deep_storage @@ -76,12 +76,9 @@ pub fn get_affinity( #[cfg(test)] mod tests { - use super::*; - - use rstest::rstest; use std::collections::BTreeMap; - use crate::DruidCluster; + use rstest::rstest; use stackable_operator::{ commons::affinity::StackableAffinity, k8s_openapi::{ @@ -92,6 +89,9 @@ mod tests { }, }; + use super::*; + use crate::crd::DruidCluster; + #[rstest] #[case(DruidRole::Coordinator)] #[case(DruidRole::Broker)] diff --git a/rust/crd/src/authentication.rs b/rust/operator-binary/src/crd/authentication.rs similarity index 99% rename from rust/crd/src/authentication.rs rename to rust/operator-binary/src/crd/authentication.rs index 537a3b22..65735a18 100644 --- a/rust/crd/src/authentication.rs +++ b/rust/operator-binary/src/crd/authentication.rs @@ -12,7 +12,7 @@ use stackable_operator::{ }; use tracing::info; -use crate::DruidClusterConfig; +use crate::crd::DruidClusterConfig; type Result = std::result::Result; @@ -232,19 +232,14 @@ impl AuthenticationClassesResolved { #[cfg(test)] mod tests { - use indoc::formatdoc; - use oidc::ClientAuthenticationOptions; - - use crate::DruidClusterConfig; - use std::pin::Pin; - use indoc::indoc; + use indoc::{formatdoc, indoc}; + use oidc::ClientAuthenticationOptions; use stackable_operator::kube; use super::*; - - use crate::authentication::AuthenticationClassesResolved; + use crate::crd::{authentication::AuthenticationClassesResolved, DruidClusterConfig}; const BASE_CLUSTER_CONFIG: &str = r#" deepStorage: diff --git a/rust/crd/src/authorization.rs b/rust/operator-binary/src/crd/authorization.rs similarity index 100% rename from rust/crd/src/authorization.rs rename to rust/operator-binary/src/crd/authorization.rs diff --git a/rust/crd/src/memory.rs b/rust/operator-binary/src/crd/memory.rs similarity index 99% rename from rust/crd/src/memory.rs rename to rust/operator-binary/src/crd/memory.rs index b4fb005c..c3da9034 100644 --- a/rust/crd/src/memory.rs +++ b/rust/operator-binary/src/crd/memory.rs @@ -7,7 +7,7 @@ use stackable_operator::{ memory::{BinaryMultiple, MemoryQuantity}, }; -use crate::{ +use crate::crd::{ storage::HistoricalStorage, PROCESSING_BUFFER_SIZE_BYTES, PROCESSING_NUM_MERGE_BUFFERS, PROCESSING_NUM_THREADS, }; @@ -167,9 +167,10 @@ fn format_for_druid(memory_quantity: &MemoryQuantity) -> String { #[cfg(test)] mod tests { - use super::*; use rstest::*; + use super::*; + #[rstest] #[case(1000, 1)] #[case(1400, 1)] diff --git a/rust/crd/src/lib.rs b/rust/operator-binary/src/crd/mod.rs similarity index 99% rename from rust/crd/src/lib.rs rename to rust/operator-binary/src/crd/mod.rs index 84a2be00..3addd72f 100644 --- a/rust/crd/src/lib.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,18 +1,4 @@ -pub mod affinity; -pub mod authentication; -pub mod authorization; -pub mod memory; -pub mod resource; -pub mod security; -pub mod storage; -pub mod tls; - -use crate::{ - affinity::get_affinity, - authorization::DruidAuthorization, - resource::RoleResource, - tls::{default_druid_tls, DruidTls}, -}; +use std::collections::{BTreeMap, HashMap, HashSet}; use indoc::formatdoc; use product_config::types::PropertyNameKind; @@ -55,9 +41,24 @@ use stackable_operator::{ COMMON_BASH_TRAP_FUNCTIONS, }, }; -use std::collections::{BTreeMap, HashMap, HashSet}; use strum::{Display, EnumDiscriminants, EnumIter, EnumString, IntoStaticStr}; +use crate::crd::{ + affinity::get_affinity, + authorization::DruidAuthorization, + resource::RoleResource, + tls::{default_druid_tls, DruidTls}, +}; + +pub mod affinity; +pub mod authentication; +pub mod authorization; +pub mod memory; +pub mod resource; +pub mod security; +pub mod storage; +pub mod tls; + pub const APP_NAME: &str = "druid"; pub const OPERATOR_NAME: &str = "druid.stackable.tech"; @@ -690,7 +691,7 @@ impl DruidCluster { Some(self.spec.cluster_config.deep_storage.to_string()), ); match self.spec.cluster_config.deep_storage.clone() { - DeepStorageSpec::HDFS(hdfs) => { + DeepStorageSpec::Hdfs(hdfs) => { result.insert(DS_DIRECTORY.to_string(), Some(hdfs.directory)); } DeepStorageSpec::S3(s3_spec) => { @@ -1076,7 +1077,7 @@ pub enum DeepStorageSpec { /// You can run an HDFS cluster with the [Stackable operator for Apache HDFS](DOCS_BASE_URL_PLACEHOLDER/hdfs/). #[serde(rename = "hdfs")] #[strum(serialize = "hdfs")] - HDFS(HdfsDeepStorageSpec), + Hdfs(HdfsDeepStorageSpec), /// [The S3 deep storage configuration](DOCS_BASE_URL_PLACEHOLDER/druid/usage-guide/deep-storage#_s3). #[strum(serialize = "s3")] S3(S3DeepStorageSpec), @@ -1084,7 +1085,7 @@ pub enum DeepStorageSpec { impl DeepStorageSpec { pub fn is_hdfs(&self) -> bool { - matches!(self, DeepStorageSpec::HDFS(_)) + matches!(self, DeepStorageSpec::Hdfs(_)) } pub fn is_s3(&self) -> bool { matches!(self, DeepStorageSpec::S3(_)) diff --git a/rust/crd/src/resource.rs b/rust/operator-binary/src/crd/resource.rs similarity index 97% rename from rust/crd/src/resource.rs rename to rust/operator-binary/src/crd/resource.rs index f122a566..9a53f019 100644 --- a/rust/crd/src/resource.rs +++ b/rust/operator-binary/src/crd/resource.rs @@ -1,13 +1,8 @@ -use std::collections::BTreeMap; -use std::sync::LazyLock; +use std::{collections::BTreeMap, sync::LazyLock}; -use crate::memory::{HistoricalDerivedSettings, RESERVED_OS_MEMORY}; -use crate::storage::{self, default_free_percentage_empty_dir_fragment}; -use crate::{DruidRole, PATH_SEGMENT_CACHE, PROP_SEGMENT_CACHE_LOCATIONS}; use snafu::{OptionExt, ResultExt, Snafu}; -use stackable_operator::builder; -use stackable_operator::memory::MemoryQuantity; use stackable_operator::{ + builder, builder::pod::{container::ContainerBuilder, volume::VolumeBuilder, PodBuilder}, commons::resources::{ CpuLimitsFragment, MemoryLimits, MemoryLimitsFragment, NoRuntimeLimits, @@ -17,9 +12,16 @@ use stackable_operator::{ api::core::v1::{EmptyDirVolumeSource, ResourceRequirements}, apimachinery::pkg::api::resource::Quantity, }, + memory::MemoryQuantity, }; use strum::{EnumDiscriminants, IntoStaticStr}; +use crate::crd::{ + memory::{HistoricalDerivedSettings, RESERVED_OS_MEMORY}, + storage::{self, default_free_percentage_empty_dir_fragment}, + DruidRole, PATH_SEGMENT_CACHE, PROP_SEGMENT_CACHE_LOCATIONS, +}; + // volume names const SEGMENT_CACHE_VOLUME_NAME: &str = "segment-cache"; @@ -29,7 +31,7 @@ const SEGMENT_CACHE_VOLUME_NAME: &str = "segment-cache"; #[allow(clippy::enum_variant_names)] pub enum Error { #[snafu(display("failed to derive Druid settings from resources"))] - DeriveMemorySettings { source: crate::memory::Error }, + DeriveMemorySettings { source: crate::crd::memory::Error }, #[snafu(display("failed to get memory limits"))] GetMemoryLimit, @@ -242,13 +244,6 @@ pub static ROUTER_RESOURCES: LazyLock Option { #[cfg(test)] mod tests { - use crate::{tests::deserialize_yaml_str, tls::DruidTls, DruidClusterConfig}; use indoc::formatdoc; + use crate::crd::{tests::deserialize_yaml_str, tls::DruidTls, DruidClusterConfig}; + const BASE_DRUID_CONFIGURATION: &str = r#" deepStorage: hdfs: diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index 0100208b..2c6b9b31 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -1,12 +1,7 @@ //! Discovery for Druid. We make Druid discoverable by putting a connection string to the router service //! inside a config map. We only provide a connection string to the router service, since it serves as //! a gateway to the cluster for client queries. -use crate::DRUID_CONTROLLER_NAME; - use snafu::{OptionExt, ResultExt, Snafu}; -use stackable_druid_crd::{ - build_recommended_labels, security::DruidTlsSecurity, DruidCluster, DruidRole, -}; use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, commons::product_image_selection::ResolvedProductImage, @@ -15,6 +10,11 @@ use stackable_operator::{ utils::cluster_info::KubernetesClusterInfo, }; +use crate::{ + crd::{build_recommended_labels, security::DruidTlsSecurity, DruidCluster, DruidRole}, + DRUID_CONTROLLER_NAME, +}; + #[derive(Snafu, Debug)] pub enum Error { #[snafu(display("object {} is missing metadata to build owner reference", druid))] diff --git a/rust/operator-binary/src/druid_controller.rs b/rust/operator-binary/src/druid_controller.rs index 0715680b..64b3bfe2 100644 --- a/rust/operator-binary/src/druid_controller.rs +++ b/rust/operator-binary/src/druid_controller.rs @@ -12,16 +12,6 @@ use product_config::{ ProductConfigManager, }; use snafu::{OptionExt, ResultExt, Snafu}; -use stackable_druid_crd::{ - authentication::AuthenticationClassesResolved, authorization::DruidAuthorization, - build_recommended_labels, build_string_list, security::DruidTlsSecurity, CommonRoleGroupConfig, - Container, DeepStorageSpec, DruidCluster, DruidClusterStatus, DruidRole, APP_NAME, - AUTH_AUTHORIZER_OPA_URI, CREDENTIALS_SECRET_PROPERTY, DB_PASSWORD_ENV, DB_USERNAME_ENV, - DRUID_CONFIG_DIRECTORY, DS_BUCKET, EXTENSIONS_LOADLIST, HDFS_CONFIG_DIRECTORY, JVM_CONFIG, - JVM_SECURITY_PROPERTIES_FILE, LOG_CONFIG_DIRECTORY, MAX_DRUID_LOG_FILES_SIZE, RUNTIME_PROPS, - RW_CONFIG_DIRECTORY, S3_ACCESS_KEY, S3_ENDPOINT_URL, S3_PATH_STYLE_ACCESS, S3_SECRET_KEY, - STACKABLE_LOG_DIR, ZOOKEEPER_CONNECTION_STRING, -}; use stackable_operator::{ builder::{ self, @@ -76,12 +66,22 @@ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ authentication::DruidAuthenticationConfig, config::get_jvm_config, + crd::{ + authentication::AuthenticationClassesResolved, authorization::DruidAuthorization, + build_recommended_labels, build_string_list, security::DruidTlsSecurity, + CommonRoleGroupConfig, Container, DeepStorageSpec, DruidCluster, DruidClusterStatus, + DruidRole, APP_NAME, AUTH_AUTHORIZER_OPA_URI, CREDENTIALS_SECRET_PROPERTY, DB_PASSWORD_ENV, + DB_USERNAME_ENV, DRUID_CONFIG_DIRECTORY, DS_BUCKET, EXTENSIONS_LOADLIST, + HDFS_CONFIG_DIRECTORY, JVM_CONFIG, JVM_SECURITY_PROPERTIES_FILE, LOG_CONFIG_DIRECTORY, + MAX_DRUID_LOG_FILES_SIZE, OPERATOR_NAME, RUNTIME_PROPS, RW_CONFIG_DIRECTORY, S3_ACCESS_KEY, + S3_ENDPOINT_URL, S3_PATH_STYLE_ACCESS, S3_SECRET_KEY, STACKABLE_LOG_DIR, + ZOOKEEPER_CONNECTION_STRING, + }, discovery::{self, build_discovery_configmaps}, extensions::get_extension_list, internal_secret::{create_shared_internal_secret, env_var_from_secret}, operations::{graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs}, product_logging::{extend_role_group_config_map, resolve_vector_aggregator_address}, - OPERATOR_NAME, }; pub const DRUID_CONTROLLER_NAME: &str = "druidcluster"; @@ -170,7 +170,7 @@ pub enum Error { }, #[snafu(display("failed to get valid S3 connection"))] - GetS3Connection { source: stackable_druid_crd::Error }, + GetS3Connection { source: crate::crd::Error }, #[snafu(display("failed to configure S3 connection"))] ConfigureS3 { source: S3Error }, @@ -220,10 +220,10 @@ pub enum Error { }, #[snafu(display("failed to resolve and merge config for role and role group"))] - FailedToResolveConfig { source: stackable_druid_crd::Error }, + FailedToResolveConfig { source: crate::crd::Error }, #[snafu(display("invalid configuration"))] - InvalidConfiguration { source: stackable_druid_crd::Error }, + InvalidConfiguration { source: crate::crd::Error }, #[snafu(display("failed to create cluster resources"))] CreateClusterResources { @@ -245,27 +245,21 @@ pub enum Error { ObjectHasNoNamespace, #[snafu(display("failed to initialize security context"))] - FailedToInitializeSecurityContext { - source: stackable_druid_crd::security::Error, - }, + FailedToInitializeSecurityContext { source: crate::crd::security::Error }, #[snafu(display("failed to retrieve AuthenticationClass"))] AuthenticationClassRetrieval { - source: stackable_druid_crd::authentication::Error, + source: crate::crd::authentication::Error, }, #[snafu(display("failed to get JVM config"))] GetJvmConfig { source: crate::config::Error }, #[snafu(display("failed to derive Druid memory settings from resources"))] - DeriveMemorySettings { - source: stackable_druid_crd::resource::Error, - }, + DeriveMemorySettings { source: crate::crd::resource::Error }, #[snafu(display("failed to update Druid config from resources"))] - UpdateDruidConfigFromResources { - source: stackable_druid_crd::resource::Error, - }, + UpdateDruidConfigFromResources { source: crate::crd::resource::Error }, #[snafu(display("failed to retrieve secret for internal communications"))] FailedInternalSecretCreation { @@ -1218,7 +1212,7 @@ fn add_hdfs_cm_volume_and_volume_mounts( pb: &mut PodBuilder, ) -> Result<()> { // hdfs deep storage mount - if let DeepStorageSpec::HDFS(hdfs) = deep_storage_spec { + if let DeepStorageSpec::Hdfs(hdfs) = deep_storage_spec { cb_druid .add_volume_mount(HDFS_CONFIG_VOLUME_NAME, HDFS_CONFIG_DIRECTORY) .context(AddVolumeMountSnafu)?; @@ -1334,11 +1328,11 @@ pub fn error_policy( #[cfg(test)] mod test { - use super::*; - use product_config::{writer, ProductConfigManager}; use rstest::*; - use stackable_druid_crd::PROP_SEGMENT_CACHE_LOCATIONS; + + use super::*; + use crate::crd::PROP_SEGMENT_CACHE_LOCATIONS; #[derive(Snafu, Debug, EnumDiscriminants)] #[strum_discriminants(derive(IntoStaticStr))] @@ -1359,9 +1353,9 @@ mod test { source: stackable_operator::product_config_utils::Error, }, #[snafu(display("failed to resolve and merge config for role and role group"))] - FailedToResolveConfig { source: stackable_druid_crd::Error }, + FailedToResolveConfig { source: crate::crd::Error }, #[snafu(display("invalid configuration"))] - InvalidConfiguration { source: stackable_druid_crd::Error }, + InvalidConfiguration { source: crate::crd::Error }, } #[rstest] diff --git a/rust/operator-binary/src/extensions.rs b/rust/operator-binary/src/extensions.rs index 847ffc6f..5212211b 100644 --- a/rust/operator-binary/src/extensions.rs +++ b/rust/operator-binary/src/extensions.rs @@ -1,9 +1,11 @@ use std::collections::HashSet; -use stackable_druid_crd::{security::DruidTlsSecurity, DbType, DruidCluster}; use tracing::debug; -use crate::authentication::DruidAuthenticationConfig; +use crate::{ + authentication::DruidAuthenticationConfig, + crd::{security::DruidTlsSecurity, DbType, DruidCluster}, +}; const EXT_S3: &str = "druid-s3-extensions"; const EXT_KAFKA_INDEXING: &str = "druid-kafka-indexing-service"; @@ -70,15 +72,13 @@ pub fn get_extension_list( #[cfg(test)] mod tests { - use stackable_druid_crd::authentication::{ - AuthenticationClassResolved, AuthenticationClassesResolved, - }; use stackable_operator::commons::{ authentication::oidc::{AuthenticationProvider, ClientAuthenticationOptions}, tls_verification::TlsClientDetails, }; use super::*; + use crate::crd::authentication::{AuthenticationClassResolved, AuthenticationClassesResolved}; #[test] fn test_additional_extensions() { diff --git a/rust/operator-binary/src/internal_secret.rs b/rust/operator-binary/src/internal_secret.rs index 677b5d41..4535bb73 100644 --- a/rust/operator-binary/src/internal_secret.rs +++ b/rust/operator-binary/src/internal_secret.rs @@ -1,14 +1,18 @@ +use std::collections::{BTreeMap, HashSet}; + use snafu::{OptionExt, ResultExt, Snafu}; -use stackable_druid_crd::security::INTERNAL_INITIAL_CLIENT_PASSWORD_ENV; -use stackable_druid_crd::{DruidCluster, COOKIE_PASSPHRASE_ENV}; -use stackable_operator::k8s_openapi::api::core::v1::{EnvVar, EnvVarSource, SecretKeySelector}; -use stackable_operator::kube::ResourceExt; use stackable_operator::{ - builder::meta::ObjectMetaBuilder, client::Client, k8s_openapi::api::core::v1::Secret, + builder::meta::ObjectMetaBuilder, + client::Client, + k8s_openapi::api::core::v1::{EnvVar, EnvVarSource, Secret, SecretKeySelector}, + kube::ResourceExt, }; -use std::collections::{BTreeMap, HashSet}; use strum::{EnumDiscriminants, IntoStaticStr}; +use crate::crd::{ + security::INTERNAL_INITIAL_CLIENT_PASSWORD_ENV, DruidCluster, COOKIE_PASSPHRASE_ENV, +}; + #[derive(Snafu, Debug, EnumDiscriminants)] #[strum_discriminants(derive(IntoStaticStr))] #[allow(clippy::enum_variant_names)] diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index b22a760c..f9526483 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -1,22 +1,10 @@ -mod authentication; -mod config; -mod discovery; -mod druid_controller; -mod extensions; -mod internal_secret; -mod operations; -mod product_logging; - use std::sync::Arc; use clap::{crate_description, crate_version, Parser}; use druid_controller::{DRUID_CONTROLLER_NAME, FULL_CONTROLLER_NAME}; use futures::StreamExt; -use stackable_druid_crd::{DruidCluster, APP_NAME, OPERATOR_NAME}; -use stackable_operator::CustomResourceExt; use stackable_operator::{ - cli::Command, - cli::ProductOperatorRun, + cli::{Command, ProductOperatorRun}, k8s_openapi::api::{ apps::v1::StatefulSet, core::v1::{ConfigMap, Service}, @@ -29,8 +17,21 @@ use stackable_operator::{ }, }, logging::controller::report_controller_reconciled, + CustomResourceExt, }; +use crate::crd::{DruidCluster, APP_NAME, OPERATOR_NAME}; + +mod authentication; +mod config; +mod crd; +mod discovery; +mod druid_controller; +mod extensions; +mod internal_secret; +mod operations; +mod product_logging; + mod built_info { include!(concat!(env!("OUT_DIR"), "/built.rs")); } diff --git a/rust/operator-binary/src/operations/graceful_shutdown.rs b/rust/operator-binary/src/operations/graceful_shutdown.rs index eabd1e8f..fe01ad8d 100644 --- a/rust/operator-binary/src/operations/graceful_shutdown.rs +++ b/rust/operator-binary/src/operations/graceful_shutdown.rs @@ -1,13 +1,13 @@ use indoc::formatdoc; use snafu::{ResultExt, Snafu}; -use stackable_druid_crd::security::DruidTlsSecurity; -use stackable_druid_crd::DruidRole; -use stackable_operator::k8s_openapi::api::core::v1::{ExecAction, LifecycleHandler}; use stackable_operator::{ builder::pod::{container::ContainerBuilder, PodBuilder}, + k8s_openapi::api::core::v1::{ExecAction, LifecycleHandler}, time::Duration, }; +use crate::crd::{security::DruidTlsSecurity, DruidRole}; + #[derive(Debug, Snafu)] pub enum Error { #[snafu(display("Failed to set terminationGracePeriod"))] diff --git a/rust/operator-binary/src/operations/pdb.rs b/rust/operator-binary/src/operations/pdb.rs index 850e4723..1c59fed7 100644 --- a/rust/operator-binary/src/operations/pdb.rs +++ b/rust/operator-binary/src/operations/pdb.rs @@ -1,11 +1,13 @@ use snafu::{ResultExt, Snafu}; -use stackable_druid_crd::{DruidCluster, DruidRole, APP_NAME, OPERATOR_NAME}; use stackable_operator::{ builder::pdb::PodDisruptionBudgetBuilder, client::Client, cluster_resources::ClusterResources, commons::pdb::PdbConfig, kube::ResourceExt, }; -use crate::druid_controller::DRUID_CONTROLLER_NAME; +use crate::{ + crd::{DruidCluster, DruidRole, APP_NAME, OPERATOR_NAME}, + druid_controller::DRUID_CONTROLLER_NAME, +}; #[derive(Snafu, Debug)] pub enum Error { diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/product_logging.rs index 22512884..fe9413f4 100644 --- a/rust/operator-binary/src/product_logging.rs +++ b/rust/operator-binary/src/product_logging.rs @@ -1,8 +1,4 @@ use snafu::{OptionExt, ResultExt, Snafu}; -use stackable_druid_crd::{ - Container, DruidCluster, DRUID_LOG_FILE, LOG4J2_CONFIG, MAX_DRUID_LOG_FILES_SIZE, - STACKABLE_LOG_DIR, -}; use stackable_operator::{ builder::configmap::ConfigMapBuilder, client::Client, @@ -16,6 +12,11 @@ use stackable_operator::{ role_utils::RoleGroupRef, }; +use crate::crd::{ + Container, DruidCluster, DRUID_LOG_FILE, LOG4J2_CONFIG, MAX_DRUID_LOG_FILES_SIZE, + STACKABLE_LOG_DIR, +}; + #[derive(Snafu, Debug)] pub enum Error { #[snafu(display("object has no namespace"))] diff --git a/rust/crd/test/resources/resource_merge/druid_cluster.yaml b/rust/operator-binary/test/resources/crd/resource_merge/druid_cluster.yaml similarity index 100% rename from rust/crd/test/resources/resource_merge/druid_cluster.yaml rename to rust/operator-binary/test/resources/crd/resource_merge/druid_cluster.yaml diff --git a/rust/crd/test/resources/resource_merge/segment_cache.yaml b/rust/operator-binary/test/resources/crd/resource_merge/segment_cache.yaml similarity index 100% rename from rust/crd/test/resources/resource_merge/segment_cache.yaml rename to rust/operator-binary/test/resources/crd/resource_merge/segment_cache.yaml diff --git a/rust/crd/test/resources/role_service/druid_cluster.yaml b/rust/operator-binary/test/resources/crd/role_service/druid_cluster.yaml similarity index 100% rename from rust/crd/test/resources/role_service/druid_cluster.yaml rename to rust/operator-binary/test/resources/crd/role_service/druid_cluster.yaml diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 00000000..455c8209 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,2 @@ +imports_granularity = "Crate" +group_imports = "StdExternalCrate" From 659ffbcd7a6d77a06745915c357be45937551e14 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 31 Jan 2025 11:22:57 +0100 Subject: [PATCH 02/10] chore: Remove unused items, add #[cfg(test)] to some items --- rust/operator-binary/src/crd/authentication.rs | 9 --------- rust/operator-binary/src/crd/mod.rs | 9 --------- rust/operator-binary/src/crd/security.rs | 1 + rust/operator-binary/src/crd/storage.rs | 3 ++- 4 files changed, 3 insertions(+), 19 deletions(-) diff --git a/rust/operator-binary/src/crd/authentication.rs b/rust/operator-binary/src/crd/authentication.rs index 65735a18..c86dde3f 100644 --- a/rust/operator-binary/src/crd/authentication.rs +++ b/rust/operator-binary/src/crd/authentication.rs @@ -219,15 +219,6 @@ impl AuthenticationClassesResolved { } false } - - pub fn oidc_authentication_enabled(&self) -> bool { - if !self.auth_classes.is_empty() { - if let Some(AuthenticationClassResolved::Oidc { .. }) = self.auth_classes.first() { - return true; - } - } - false - } } #[cfg(test)] diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 3addd72f..ef5d8f63 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -77,7 +77,6 @@ pub const JVM_SECURITY_PROPERTIES_FILE: &str = "security.properties"; // store directories pub const STACKABLE_TRUST_STORE: &str = "/stackable/truststore.p12"; pub const STACKABLE_TRUST_STORE_PASSWORD: &str = "changeit"; -pub const CERTS_DIR: &str = "/stackable/certificates"; pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; // store file names @@ -132,14 +131,6 @@ pub const MAX_DRUID_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { // metrics pub const PROMETHEUS_PORT: &str = "druid.emitter.prometheus.port"; pub const METRICS_PORT: u16 = 9090; -// container locations -pub const S3_SECRET_DIR_NAME: &str = "/stackable/secrets"; -pub const SECRET_KEY_S3_ACCESS_KEY: &str = "accessKey"; -pub const SECRET_KEY_S3_SECRET_KEY: &str = "secretKey"; -// segment storage -pub const SC_LOCATIONS: &str = "druid.segmentCache.locations"; -pub const SC_DIRECTORY: &str = "/stackable/var/druid/segment-cache"; -pub const SC_VOLUME_NAME: &str = "segment-cache"; pub const COOKIE_PASSPHRASE_ENV: &str = "OIDC_COOKIE_PASSPHRASE"; diff --git a/rust/operator-binary/src/crd/security.rs b/rust/operator-binary/src/crd/security.rs index fc97fd69..5db33a02 100644 --- a/rust/operator-binary/src/crd/security.rs +++ b/rust/operator-binary/src/crd/security.rs @@ -103,6 +103,7 @@ pub const INTERNAL_INITIAL_CLIENT_PASSWORD_ENV: &str = "INTERNAL_INITIAL_CLIENT_ pub const ESCALATOR_INTERNAL_CLIENT_PASSWORD_ENV: &str = INTERNAL_INITIAL_CLIENT_PASSWORD_ENV; impl DruidTlsSecurity { + #[cfg(test)] pub fn new( auth_classes: &AuthenticationClassesResolved, server_and_internal_secret_class: Option, diff --git a/rust/operator-binary/src/crd/storage.rs b/rust/operator-binary/src/crd/storage.rs index a081bf0d..16e0da95 100644 --- a/rust/operator-binary/src/crd/storage.rs +++ b/rust/operator-binary/src/crd/storage.rs @@ -1,7 +1,7 @@ use serde::{Deserialize, Serialize}; -use stackable_operator::k8s_openapi::apimachinery::pkg::api::resource::Quantity; use stackable_operator::{ config::{fragment::Fragment, merge::Merge}, + k8s_openapi::apimachinery::pkg::api::resource::Quantity, schemars::{self, JsonSchema}, }; @@ -71,6 +71,7 @@ pub struct FreePercentageEmptyDir { pub empty_dir: CapacityEmptyDir, } +#[cfg(test)] pub fn default_free_percentage_empty_dir() -> FreePercentageEmptyDir { FreePercentageEmptyDir { free_percentage: Some(5), From 4fdad3e0b46fc7f6bfcea48d8cb8bae3ea17ed05 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 31 Jan 2025 11:27:56 +0100 Subject: [PATCH 03/10] test: Fix unit tests --- rust/operator-binary/src/crd/mod.rs | 5 +++-- rust/operator-binary/src/crd/resource.rs | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index ef5d8f63..96b20f79 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1563,8 +1563,9 @@ mod tests { #[test] fn test_service_name_generation() { - let cluster = - deserialize_yaml_file::("test/resources/role_service/druid_cluster.yaml"); + let cluster = deserialize_yaml_file::( + "test/resources/crd/role_service/druid_cluster.yaml", + ); let dummy_cluster_info = KubernetesClusterInfo { cluster_domain: DomainName::try_from("cluster.local").unwrap(), }; diff --git a/rust/operator-binary/src/crd/resource.rs b/rust/operator-binary/src/crd/resource.rs index 9a53f019..a3565372 100644 --- a/rust/operator-binary/src/crd/resource.rs +++ b/rust/operator-binary/src/crd/resource.rs @@ -406,7 +406,7 @@ mod test { #[test] fn test_resources() -> Result<(), Error> { let cluster = deserialize_yaml_file::( - "test/resources/resource_merge/druid_cluster.yaml", + "test/resources/crd/resource_merge/druid_cluster.yaml", ); let config = cluster.merged_config().unwrap(); @@ -482,7 +482,7 @@ mod test { #[test] fn test_segment_cache() -> Result<(), Error> { let cluster = deserialize_yaml_file::( - "test/resources/resource_merge/segment_cache.yaml", + "test/resources/crd/resource_merge/segment_cache.yaml", ); // ---------- default role group From 5720c2130456048b4208490e16589292e3fd85ec Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 31 Jan 2025 12:10:03 +0100 Subject: [PATCH 04/10] chore: Version DruidCluster --- Cargo.lock | 300 ++++++++++++++++-- Cargo.toml | 1 + rust/operator-binary/Cargo.toml | 6 +- .../operator-binary/src/authentication/mod.rs | 8 +- rust/operator-binary/src/crd/affinity.rs | 4 +- rust/operator-binary/src/crd/mod.rs | 115 +++---- rust/operator-binary/src/crd/resource.rs | 8 +- rust/operator-binary/src/crd/security.rs | 4 +- rust/operator-binary/src/discovery.rs | 8 +- rust/operator-binary/src/druid_controller.rs | 47 ++- rust/operator-binary/src/extensions.rs | 9 +- rust/operator-binary/src/internal_secret.rs | 12 +- rust/operator-binary/src/main.rs | 10 +- rust/operator-binary/src/operations/pdb.rs | 4 +- rust/operator-binary/src/product_logging.rs | 7 +- rustfmt.toml | 2 + 16 files changed, 403 insertions(+), 142 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 35277d06..97c2a10a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -220,6 +220,12 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.8.0" @@ -368,6 +374,15 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -624,6 +639,15 @@ dependencies = [ "regex-syntax 0.8.5", ] +[[package]] +name = "fluent-uri" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17c704e9dbe1ddd863da1e6ff3567795087b1eb201ce80d8fa81162e1516500d" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "fnv" version = "1.0.7" @@ -795,7 +819,7 @@ version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b903b73e45dc0c6c596f2d37eccece7c1c8bb6e4407b001096387c63d0d93724" dependencies = [ - "bitflags", + "bitflags 2.8.0", "libc", "libgit2-sys", "log", @@ -808,6 +832,16 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + [[package]] name = "hashbrown" version = "0.15.2" @@ -1186,7 +1220,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.15.2", ] [[package]] @@ -1216,6 +1250,15 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.14" @@ -1252,18 +1295,45 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json-patch" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b1fb8864823fad91877e6caea0baca82e49e8db50f8e5c9f9a453e27d3330fc" +dependencies = [ + "jsonptr 0.4.7", + "serde", + "serde_json", + "thiserror 1.0.69", +] + [[package]] name = "json-patch" version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "863726d7afb6bc2590eeff7135d923545e5e964f004c2ccf8716c25e70a86f08" dependencies = [ - "jsonptr", + "jsonptr 0.6.3", "serde", "serde_json", "thiserror 1.0.69", ] +[[package]] +name = "jsonpath-rust" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d8fe85bd70ff715f31ce8c739194b423d79811a19602115d611a3ec85d6200" +dependencies = [ + "lazy_static", + "once_cell", + "pest", + "pest_derive", + "regex", + "serde_json", + "thiserror 1.0.69", +] + [[package]] name = "jsonpath-rust" version = "0.7.5" @@ -1277,6 +1347,17 @@ dependencies = [ "thiserror 2.0.11", ] +[[package]] +name = "jsonptr" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c6e529149475ca0b2820835d3dce8fcc41c6b943ca608d32f35b449255e4627" +dependencies = [ + "fluent-uri", + "serde", + "serde_json", +] + [[package]] name = "jsonptr" version = "0.6.3" @@ -1287,6 +1368,20 @@ dependencies = [ "serde_json", ] +[[package]] +name = "k8s-openapi" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8847402328d8301354c94d605481f25a6bdc1ed65471fd96af8eca71141b13" +dependencies = [ + "base64 0.22.1", + "chrono", + "schemars", + "serde", + "serde-value", + "serde_json", +] + [[package]] name = "k8s-openapi" version = "0.24.0" @@ -1301,17 +1396,78 @@ dependencies = [ "serde_json", ] +[[package]] +name = "k8s-version" +version = "0.1.2" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-versioned-0.5.0#048c7d8befddc2f2c6414444006871c95412d67c" +dependencies = [ + "darling", + "regex", + "snafu 0.8.5", +] + +[[package]] +name = "kube" +version = "0.96.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efffeb3df0bd4ef3e5d65044573499c0e4889b988070b08c50b25b1329289a1f" +dependencies = [ + "k8s-openapi 0.23.0", + "kube-client 0.96.0", + "kube-core 0.96.0", + "kube-derive 0.96.0", + "kube-runtime 0.96.0", +] + [[package]] name = "kube" version = "0.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32053dc495efad4d188c7b33cc7c02ef4a6e43038115348348876efd39a53cba" dependencies = [ - "k8s-openapi", - "kube-client", - "kube-core", - "kube-derive", - "kube-runtime", + "k8s-openapi 0.24.0", + "kube-client 0.98.0", + "kube-core 0.98.0", + "kube-derive 0.98.0", + "kube-runtime 0.98.0", +] + +[[package]] +name = "kube-client" +version = "0.96.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bf471ece8ff8d24735ce78dac4d091e9fcb8d74811aeb6b75de4d1c3f5de0f1" +dependencies = [ + "base64 0.22.1", + "bytes", + "chrono", + "either", + "futures 0.3.31", + "home", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-http-proxy", + "hyper-rustls", + "hyper-timeout", + "hyper-util", + "jsonpath-rust 0.5.1", + "k8s-openapi 0.23.0", + "kube-core 0.96.0", + "pem", + "rustls", + "rustls-pemfile", + "secrecy", + "serde", + "serde_json", + "serde_yaml", + "thiserror 1.0.69", + "tokio", + "tokio-util", + "tower", + "tower-http", + "tracing", ] [[package]] @@ -1334,9 +1490,9 @@ dependencies = [ "hyper-rustls", "hyper-timeout", "hyper-util", - "jsonpath-rust", - "k8s-openapi", - "kube-core", + "jsonpath-rust 0.7.5", + "k8s-openapi 0.24.0", + "kube-core 0.98.0", "pem", "rustls", "rustls-pemfile", @@ -1352,6 +1508,24 @@ dependencies = [ "tracing", ] +[[package]] +name = "kube-core" +version = "0.96.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f42346d30bb34d1d7adc5c549b691bce7aa3a1e60254e68fab7e2d7b26fe3d77" +dependencies = [ + "chrono", + "form_urlencoded", + "http", + "json-patch 2.0.0", + "k8s-openapi 0.23.0", + "schemars", + "serde", + "serde-value", + "serde_json", + "thiserror 1.0.69", +] + [[package]] name = "kube-core" version = "0.98.0" @@ -1361,8 +1535,8 @@ dependencies = [ "chrono", "form_urlencoded", "http", - "json-patch", - "k8s-openapi", + "json-patch 3.0.1", + "k8s-openapi 0.24.0", "schemars", "serde", "serde-value", @@ -1370,6 +1544,19 @@ dependencies = [ "thiserror 2.0.11", ] +[[package]] +name = "kube-derive" +version = "0.96.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9364e04cc5e0482136c6ee8b7fb7551812da25802249f35b3def7aaa31e82ad" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "serde_json", + "syn 2.0.96", +] + [[package]] name = "kube-derive" version = "0.98.0" @@ -1383,6 +1570,34 @@ dependencies = [ "syn 2.0.96", ] +[[package]] +name = "kube-runtime" +version = "0.96.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fbf1f6ffa98e65f1d2a9a69338bb60605d46be7edf00237784b89e62c9bd44" +dependencies = [ + "ahash", + "async-broadcast", + "async-stream", + "async-trait", + "backoff", + "educe", + "futures 0.3.31", + "hashbrown 0.14.5", + "json-patch 2.0.0", + "jsonptr 0.4.7", + "k8s-openapi 0.23.0", + "kube-client 0.96.0", + "parking_lot", + "pin-project", + "serde", + "serde_json", + "thiserror 1.0.69", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "kube-runtime" version = "0.98.0" @@ -1396,12 +1611,12 @@ dependencies = [ "backoff", "educe", "futures 0.3.31", - "hashbrown", + "hashbrown 0.15.2", "hostname", - "json-patch", - "jsonptr", - "k8s-openapi", - "kube-client", + "json-patch 3.0.1", + "jsonptr 0.6.3", + "k8s-openapi 0.24.0", + "kube-client 0.98.0", "parking_lot", "pin-project", "serde", @@ -1567,7 +1782,7 @@ version = "0.10.68" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" dependencies = [ - "bitflags", + "bitflags 2.8.0", "cfg-if", "foreign-types", "libc", @@ -1909,7 +2124,7 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ - "bitflags", + "bitflags 2.8.0", ] [[package]] @@ -2155,7 +2370,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags", + "bitflags 2.8.0", "core-foundation 0.9.4", "core-foundation-sys", "libc", @@ -2168,7 +2383,7 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ - "bitflags", + "bitflags 2.8.0", "core-foundation 0.10.0", "core-foundation-sys", "libc", @@ -2403,6 +2618,7 @@ dependencies = [ "serde_yaml", "snafu 0.8.5", "stackable-operator", + "stackable-versioned", "strum", "tokio", "tracing", @@ -2422,9 +2638,9 @@ dependencies = [ "either", "futures 0.3.31", "indexmap", - "json-patch", - "k8s-openapi", - "kube", + "json-patch 3.0.1", + "k8s-openapi 0.24.0", + "kube 0.98.0", "opentelemetry-jaeger", "opentelemetry_sdk", "product-config", @@ -2462,13 +2678,37 @@ name = "stackable-shared" version = "0.0.1" source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#59506c6202778889a27b6ae8153457e60a49c68d" dependencies = [ - "kube", + "kube 0.98.0", "semver", "serde", "serde_yaml", "snafu 0.8.5", ] +[[package]] +name = "stackable-versioned" +version = "0.5.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-versioned-0.5.0#048c7d8befddc2f2c6414444006871c95412d67c" +dependencies = [ + "stackable-versioned-macros", +] + +[[package]] +name = "stackable-versioned-macros" +version = "0.5.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-versioned-0.5.0#048c7d8befddc2f2c6414444006871c95412d67c" +dependencies = [ + "convert_case", + "darling", + "itertools", + "k8s-openapi 0.23.0", + "k8s-version", + "kube 0.96.0", + "proc-macro2", + "quote", + "syn 2.0.96", +] + [[package]] name = "strsim" version = "0.11.1" @@ -2760,7 +3000,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697" dependencies = [ "base64 0.22.1", - "bitflags", + "bitflags 2.8.0", "bytes", "http", "http-body", @@ -2912,6 +3152,12 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11cd88e12b17c6494200a9c1b683a04fcac9573ed74cd1b62aeb2727c5592243" +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + [[package]] name = "unicode-xid" version = "0.2.6" diff --git a/Cargo.toml b/Cargo.toml index d48513df..c08c9715 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ edition = "2021" repository = "https://github.com/stackabletech/druid-operator" [workspace.dependencies] +stackable-versioned = { git = "https://github.com/stackabletech/operator-rs.git", features = ["k8s"], tag = "stackable-versioned-0.5.0" } stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.85.0" } product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.7.0" } diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 5eaa4564..8e93b968 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -9,6 +9,10 @@ repository.workspace = true publish = false [dependencies] +stackable-versioned.workspace = true +stackable-operator.workspace = true +product-config.workspace = true + anyhow.workspace = true clap.workspace = true const_format.workspace = true @@ -21,8 +25,6 @@ semver.workspace = true serde.workspace = true serde_json.workspace = true snafu.workspace = true -stackable-operator.workspace = true -product-config.workspace = true strum.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/rust/operator-binary/src/authentication/mod.rs b/rust/operator-binary/src/authentication/mod.rs index 8a92b69a..2c1e31e0 100644 --- a/rust/operator-binary/src/authentication/mod.rs +++ b/rust/operator-binary/src/authentication/mod.rs @@ -19,7 +19,7 @@ use crate::{ crd::{ authentication::{AuthenticationClassResolved, AuthenticationClassesResolved}, security::{ESCALATOR_INTERNAL_CLIENT_PASSWORD_ENV, INTERNAL_INITIAL_CLIENT_PASSWORD_ENV}, - DruidCluster, DruidRole, + v1alpha1, DruidRole, }, internal_secret::{build_shared_internal_secret_name, env_var_from_secret}, }; @@ -161,7 +161,11 @@ impl DruidAuthenticationConfig { command } - pub fn get_env_var_mounts(&self, druid: &DruidCluster, role: &DruidRole) -> Vec { + pub fn get_env_var_mounts( + &self, + druid: &v1alpha1::DruidCluster, + role: &DruidRole, + ) -> Vec { let mut envs = vec![]; let internal_secret_name = build_shared_internal_secret_name(druid); envs.push(env_var_from_secret( diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index 6f0b3b9c..cfbc254d 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -90,7 +90,7 @@ mod tests { }; use super::*; - use crate::crd::DruidCluster; + use crate::crd::v1alpha1; #[rstest] #[case(DruidRole::Coordinator)] @@ -141,7 +141,7 @@ mod tests { replicas: 1 "#; let deserializer = serde_yaml::Deserializer::from_str(input); - let druid: DruidCluster = + let druid: v1alpha1::DruidCluster = serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); let merged_config = druid .merged_config() diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 96b20f79..98ce127e 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -41,6 +41,7 @@ use stackable_operator::{ COMMON_BASH_TRAP_FUNCTIONS, }, }; +use stackable_versioned::versioned; use strum::{Display, EnumDiscriminants, EnumIter, EnumString, IntoStaticStr}; use crate::crd::{ @@ -179,50 +180,52 @@ pub enum Error { FragmentValidationFailure { source: ValidationError }, } -/// A Druid cluster stacklet. This resource is managed by the Stackable operator for Apache Druid. -/// Find more information on how to use it and the resources that the operator generates in the -/// [operator documentation](DOCS_BASE_URL_PLACEHOLDER/druid/). -#[derive(Clone, CustomResource, Debug, Deserialize, JsonSchema, Serialize)] -#[kube( - group = "druid.stackable.tech", - version = "v1alpha1", - kind = "DruidCluster", - plural = "druidclusters", - shortname = "druid", - status = "DruidClusterStatus", - namespaced, - crates( - kube_core = "stackable_operator::kube::core", - k8s_openapi = "stackable_operator::k8s_openapi", - schemars = "stackable_operator::schemars" - ) -)] -#[serde(rename_all = "camelCase")] -pub struct DruidClusterSpec { - /// Common cluster wide configuration that can not differ or be overridden on a role or role group level. - pub cluster_config: DruidClusterConfig, +#[versioned(version(name = "v1alpha1"))] +pub mod versioned { + /// A Druid cluster stacklet. This resource is managed by the Stackable operator for Apache Druid. + /// Find more information on how to use it and the resources that the operator generates in the + /// [operator documentation](DOCS_BASE_URL_PLACEHOLDER/druid/). + #[versioned(k8s( + group = "druid.stackable.tech", + kind = "DruidCluster", + plural = "druidclusters", + shortname = "druid", + status = "DruidClusterStatus", + namespaced, + crates( + kube_core = "stackable_operator::kube::core", + k8s_openapi = "stackable_operator::k8s_openapi", + schemars = "stackable_operator::schemars" + ) + ))] + #[derive(Clone, CustomResource, Debug, Deserialize, JsonSchema, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct DruidClusterSpec { + /// Common cluster wide configuration that can not differ or be overridden on a role or role group level. + pub cluster_config: DruidClusterConfig, - // no doc - docs provided by the struct. - pub image: ProductImage, + // no doc - docs provided by the struct. + pub image: ProductImage, - // no doc - docs provided by the struct. - pub brokers: Role, + // no doc - docs provided by the struct. + pub brokers: Role, - // no doc - docs provided by the struct. - pub coordinators: Role, + // no doc - docs provided by the struct. + pub coordinators: Role, - // no doc - docs provided by the struct. - pub historicals: Role, + // no doc - docs provided by the struct. + pub historicals: Role, - // no doc - docs provided by the struct. - pub middle_managers: Role, + // no doc - docs provided by the struct. + pub middle_managers: Role, - // no doc - docs provided by the struct. - pub routers: Role, + // no doc - docs provided by the struct. + pub routers: Role, - // no doc - docs provided by the struct. - #[serde(default)] - pub cluster_operation: ClusterOperation, + // no doc - docs provided by the struct. + #[serde(default)] + pub cluster_operation: ClusterOperation, + } } #[derive( @@ -617,7 +620,7 @@ impl DruidRole { } // Required to retrieve the conditions from the cluster status -impl HasStatusCondition for DruidCluster { +impl HasStatusCondition for v1alpha1::DruidCluster { fn conditions(&self) -> Vec { match &self.status { Some(status) => status.conditions.clone(), @@ -626,7 +629,7 @@ impl HasStatusCondition for DruidCluster { } } -impl DruidCluster { +impl v1alpha1::DruidCluster { pub fn common_compute_files( &self, file: &str, @@ -709,7 +712,7 @@ impl DruidCluster { String, ( Vec, - Role>, + Role>, ), > { let config_files = vec![ @@ -856,11 +859,11 @@ impl DruidCluster { let deep_storage = &self.spec.cluster_config.deep_storage; Ok(MergedConfig { - brokers: DruidCluster::merged_role( + brokers: v1alpha1::DruidCluster::merged_role( &self.spec.brokers, &BrokerConfig::default_config(&self.name_any(), &DruidRole::Broker, deep_storage), )?, - coordinators: DruidCluster::merged_role( + coordinators: v1alpha1::DruidCluster::merged_role( &self.spec.coordinators, &CoordinatorConfig::default_config( &self.name_any(), @@ -868,7 +871,7 @@ impl DruidCluster { deep_storage, ), )?, - historicals: DruidCluster::merged_role( + historicals: v1alpha1::DruidCluster::merged_role( &self.spec.historicals, &HistoricalConfig::default_config( &self.name_any(), @@ -876,7 +879,7 @@ impl DruidCluster { deep_storage, ), )?, - middle_managers: DruidCluster::merged_role( + middle_managers: v1alpha1::DruidCluster::merged_role( &self.spec.middle_managers, &MiddleManagerConfig::default_config( &self.name_any(), @@ -884,7 +887,7 @@ impl DruidCluster { deep_storage, ), )?, - routers: DruidCluster::merged_role( + routers: v1alpha1::DruidCluster::merged_role( &self.spec.routers, &RouterConfig::default_config(&self.name_any(), &DruidRole::Router, deep_storage), )?, @@ -903,8 +906,11 @@ impl DruidCluster { let mut merged_role_config = HashMap::new(); for (rolegroup_name, rolegroup) in &role.role_groups { - let merged_rolegroup_config = - DruidCluster::merged_rolegroup(rolegroup, &role.config.config, default_config)?; + let merged_rolegroup_config = v1alpha1::DruidCluster::merged_rolegroup( + rolegroup, + &role.config.config, + default_config, + )?; merged_role_config.insert(rolegroup_name.to_owned(), merged_rolegroup_config); } @@ -921,7 +927,7 @@ impl DruidCluster { T: FromFragment, T::Fragment: Clone + Merge, { - let merged_config = DruidCluster::merged_rolegroup_config( + let merged_config = v1alpha1::DruidCluster::merged_rolegroup_config( &rolegroup.config.config, role_config, default_config, @@ -1078,6 +1084,7 @@ impl DeepStorageSpec { pub fn is_hdfs(&self) -> bool { matches!(self, DeepStorageSpec::Hdfs(_)) } + pub fn is_s3(&self) -> bool { matches!(self, DeepStorageSpec::S3(_)) } @@ -1363,7 +1370,7 @@ impl HistoricalConfig { } impl Configuration for BrokerConfigFragment { - type Configurable = DruidCluster; + type Configurable = v1alpha1::DruidCluster; fn compute_env( &self, @@ -1393,7 +1400,7 @@ impl Configuration for BrokerConfigFragment { } impl Configuration for HistoricalConfigFragment { - type Configurable = DruidCluster; + type Configurable = v1alpha1::DruidCluster; fn compute_env( &self, @@ -1423,7 +1430,7 @@ impl Configuration for HistoricalConfigFragment { } impl Configuration for RouterConfigFragment { - type Configurable = DruidCluster; + type Configurable = v1alpha1::DruidCluster; fn compute_env( &self, @@ -1453,7 +1460,7 @@ impl Configuration for RouterConfigFragment { } impl Configuration for MiddleManagerConfigFragment { - type Configurable = DruidCluster; + type Configurable = v1alpha1::DruidCluster; fn compute_env( &self, @@ -1492,7 +1499,7 @@ impl Configuration for MiddleManagerConfigFragment { } impl Configuration for CoordinatorConfigFragment { - type Configurable = DruidCluster; + type Configurable = v1alpha1::DruidCluster; fn compute_env( &self, @@ -1563,7 +1570,7 @@ mod tests { #[test] fn test_service_name_generation() { - let cluster = deserialize_yaml_file::( + let cluster = deserialize_yaml_file::( "test/resources/crd/role_service/druid_cluster.yaml", ); let dummy_cluster_info = KubernetesClusterInfo { diff --git a/rust/operator-binary/src/crd/resource.rs b/rust/operator-binary/src/crd/resource.rs index a3565372..a8454a3a 100644 --- a/rust/operator-binary/src/crd/resource.rs +++ b/rust/operator-binary/src/crd/resource.rs @@ -258,7 +258,7 @@ mod test { use crate::crd::{ storage::{default_free_percentage_empty_dir, HistoricalStorage}, tests::deserialize_yaml_file, - DruidCluster, MiddleManagerConfig, + v1alpha1, MiddleManagerConfig, }; #[rstest] @@ -394,7 +394,7 @@ mod test { #[case] third: Option>, #[case] expected: Resources, ) { - let got = DruidCluster::merged_rolegroup_config( + let got = v1alpha1::DruidCluster::merged_rolegroup_config( &first.unwrap_or_default(), &second.unwrap_or_default(), &third.unwrap_or_default(), @@ -405,7 +405,7 @@ mod test { #[test] fn test_resources() -> Result<(), Error> { - let cluster = deserialize_yaml_file::( + let cluster = deserialize_yaml_file::( "test/resources/crd/resource_merge/druid_cluster.yaml", ); @@ -481,7 +481,7 @@ mod test { #[test] fn test_segment_cache() -> Result<(), Error> { - let cluster = deserialize_yaml_file::( + let cluster = deserialize_yaml_file::( "test/resources/crd/resource_merge/segment_cache.yaml", ); diff --git a/rust/operator-binary/src/crd/security.rs b/rust/operator-binary/src/crd/security.rs index 5db33a02..801bcd75 100644 --- a/rust/operator-binary/src/crd/security.rs +++ b/rust/operator-binary/src/crd/security.rs @@ -22,7 +22,7 @@ use stackable_operator::{ use crate::crd::{ authentication::{self, AuthenticationClassesResolved}, - DruidCluster, DruidRole, METRICS_PORT, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD, + v1alpha1, DruidRole, METRICS_PORT, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD, }; #[derive(Snafu, Debug)] @@ -117,7 +117,7 @@ impl DruidTlsSecurity { /// Create a `DruidTlsSecurity` struct from the Druid custom resource and resolve /// all provided `AuthenticationClass` references. pub fn new_from_druid_cluster( - druid: &DruidCluster, + druid: &v1alpha1::DruidCluster, auth_classes: &AuthenticationClassesResolved, ) -> Self { DruidTlsSecurity { diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index 2c6b9b31..fcd6997c 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -11,7 +11,7 @@ use stackable_operator::{ }; use crate::{ - crd::{build_recommended_labels, security::DruidTlsSecurity, DruidCluster, DruidRole}, + crd::{build_recommended_labels, security::DruidTlsSecurity, v1alpha1, DruidRole}, DRUID_CONTROLLER_NAME, }; @@ -20,7 +20,7 @@ pub enum Error { #[snafu(display("object {} is missing metadata to build owner reference", druid))] ObjectMissingMetadataForOwnerRef { source: stackable_operator::builder::meta::Error, - druid: ObjectRef, + druid: ObjectRef, }, #[snafu(display("failed to get service FQDN"))] @@ -39,7 +39,7 @@ pub enum Error { /// Builds discovery [`ConfigMap`]s for connecting to a [`DruidCluster`] pub async fn build_discovery_configmaps( - druid: &DruidCluster, + druid: &v1alpha1::DruidCluster, owner: &impl Resource, cluster_info: &KubernetesClusterInfo, resolved_product_image: &ResolvedProductImage, @@ -58,7 +58,7 @@ pub async fn build_discovery_configmaps( /// Build a discovery [`ConfigMap`] containing information about how to connect to a certain [`DruidCluster`] fn build_discovery_configmap( - druid: &DruidCluster, + druid: &v1alpha1::DruidCluster, owner: &impl Resource, cluster_info: &KubernetesClusterInfo, resolved_product_image: &ResolvedProductImage, diff --git a/rust/operator-binary/src/druid_controller.rs b/rust/operator-binary/src/druid_controller.rs index 64b3bfe2..7d5d0f4b 100644 --- a/rust/operator-binary/src/druid_controller.rs +++ b/rust/operator-binary/src/druid_controller.rs @@ -68,14 +68,13 @@ use crate::{ config::get_jvm_config, crd::{ authentication::AuthenticationClassesResolved, authorization::DruidAuthorization, - build_recommended_labels, build_string_list, security::DruidTlsSecurity, - CommonRoleGroupConfig, Container, DeepStorageSpec, DruidCluster, DruidClusterStatus, - DruidRole, APP_NAME, AUTH_AUTHORIZER_OPA_URI, CREDENTIALS_SECRET_PROPERTY, DB_PASSWORD_ENV, - DB_USERNAME_ENV, DRUID_CONFIG_DIRECTORY, DS_BUCKET, EXTENSIONS_LOADLIST, - HDFS_CONFIG_DIRECTORY, JVM_CONFIG, JVM_SECURITY_PROPERTIES_FILE, LOG_CONFIG_DIRECTORY, - MAX_DRUID_LOG_FILES_SIZE, OPERATOR_NAME, RUNTIME_PROPS, RW_CONFIG_DIRECTORY, S3_ACCESS_KEY, - S3_ENDPOINT_URL, S3_PATH_STYLE_ACCESS, S3_SECRET_KEY, STACKABLE_LOG_DIR, - ZOOKEEPER_CONNECTION_STRING, + build_recommended_labels, build_string_list, security::DruidTlsSecurity, v1alpha1, + CommonRoleGroupConfig, Container, DeepStorageSpec, DruidClusterStatus, DruidRole, APP_NAME, + AUTH_AUTHORIZER_OPA_URI, CREDENTIALS_SECRET_PROPERTY, DB_PASSWORD_ENV, DB_USERNAME_ENV, + DRUID_CONFIG_DIRECTORY, DS_BUCKET, EXTENSIONS_LOADLIST, HDFS_CONFIG_DIRECTORY, JVM_CONFIG, + JVM_SECURITY_PROPERTIES_FILE, LOG_CONFIG_DIRECTORY, MAX_DRUID_LOG_FILES_SIZE, + OPERATOR_NAME, RUNTIME_PROPS, RW_CONFIG_DIRECTORY, S3_ACCESS_KEY, S3_ENDPOINT_URL, + S3_PATH_STYLE_ACCESS, S3_SECRET_KEY, STACKABLE_LOG_DIR, ZOOKEEPER_CONNECTION_STRING, }, discovery::{self, build_discovery_configmaps}, extensions::get_extension_list, @@ -115,25 +114,25 @@ pub enum Error { #[snafu(display("failed to apply Service for {}", rolegroup))] ApplyRoleGroupService { source: stackable_operator::cluster_resources::Error, - rolegroup: RoleGroupRef, + rolegroup: RoleGroupRef, }, #[snafu(display("failed to build ConfigMap for {}", rolegroup))] BuildRoleGroupConfig { source: stackable_operator::builder::configmap::Error, - rolegroup: RoleGroupRef, + rolegroup: RoleGroupRef, }, #[snafu(display("failed to apply ConfigMap for {}", rolegroup))] ApplyRoleGroupConfig { source: stackable_operator::cluster_resources::Error, - rolegroup: RoleGroupRef, + rolegroup: RoleGroupRef, }, #[snafu(display("failed to apply StatefulSet for {}", rolegroup))] ApplyRoleGroupStatefulSet { source: stackable_operator::cluster_resources::Error, - rolegroup: RoleGroupRef, + rolegroup: RoleGroupRef, }, #[snafu(display("invalid product configuration"))] @@ -370,7 +369,7 @@ impl ReconcilerError for Error { } pub async fn reconcile_druid( - druid: Arc>, + druid: Arc>, ctx: Arc, ) -> Result { tracing::info!("Starting reconcile"); @@ -632,7 +631,7 @@ pub async fn reconcile_druid( /// The server-role service is the primary endpoint that should be used by clients that do not perform internal load balancing, /// including targets outside of the cluster. pub fn build_role_service( - druid: &DruidCluster, + druid: &v1alpha1::DruidCluster, resolved_product_image: &ResolvedProductImage, role: &DruidRole, druid_tls_security: &DruidTlsSecurity, @@ -675,9 +674,9 @@ pub fn build_role_service( #[allow(clippy::too_many_arguments)] /// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator fn build_rolegroup_config_map( - druid: &DruidCluster, + druid: &v1alpha1::DruidCluster, resolved_product_image: &ResolvedProductImage, - rolegroup: &RoleGroupRef, + rolegroup: &RoleGroupRef, rolegroup_config: &HashMap>, merged_rolegroup_config: &CommonRoleGroupConfig, zk_connstr: &str, @@ -855,9 +854,9 @@ fn build_rolegroup_config_map( /// /// This is mostly useful for internal communication between peers, or for clients that perform client-side load balancing. fn build_rolegroup_services( - druid: &DruidCluster, + druid: &v1alpha1::DruidCluster, resolved_product_image: &ResolvedProductImage, - rolegroup: &RoleGroupRef, + rolegroup: &RoleGroupRef, druid_tls_security: &DruidTlsSecurity, ) -> Result { let role = DruidRole::from_str(&rolegroup.role).unwrap(); @@ -905,10 +904,10 @@ fn build_rolegroup_services( /// /// The [`Pod`](`stackable_operator::k8s_openapi::api::core::v1::Pod`)s are accessible through the corresponding [`Service`] (from [`build_rolegroup_services`]). fn build_rolegroup_statefulset( - druid: &DruidCluster, + druid: &v1alpha1::DruidCluster, resolved_product_image: &ResolvedProductImage, role: &DruidRole, - rolegroup_ref: &RoleGroupRef, + rolegroup_ref: &RoleGroupRef, rolegroup_config: &HashMap>, merged_rolegroup_config: &CommonRoleGroupConfig, s3_conn: Option<&S3ConnectionSpec>, @@ -1228,7 +1227,7 @@ fn add_hdfs_cm_volume_and_volume_mounts( } fn add_config_volume_and_volume_mounts( - rolegroup_ref: &RoleGroupRef, + rolegroup_ref: &RoleGroupRef, cb_druid: &mut ContainerBuilder, pb: &mut PodBuilder, ) -> Result<()> { @@ -1255,7 +1254,7 @@ fn add_config_volume_and_volume_mounts( } fn add_log_config_volume_and_volume_mounts( - rolegroup_ref: &RoleGroupRef, + rolegroup_ref: &RoleGroupRef, merged_rolegroup_config: &CommonRoleGroupConfig, cb_druid: &mut ContainerBuilder, pb: &mut PodBuilder, @@ -1316,7 +1315,7 @@ fn add_log_volume_and_volume_mounts( } pub fn error_policy( - _obj: Arc>, + _obj: Arc>, error: &Error, _ctx: Arc, ) -> Action { @@ -1378,7 +1377,7 @@ mod test { std::fs::File::open(format!("test/resources/druid_controller/{druid_manifest}")) .unwrap(); let deserializer = serde_yaml::Deserializer::from_reader(&cluster_cr); - let druid: DruidCluster = + let druid: v1alpha1::DruidCluster = serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); let resolved_product_image: ResolvedProductImage = druid diff --git a/rust/operator-binary/src/extensions.rs b/rust/operator-binary/src/extensions.rs index 5212211b..837108e4 100644 --- a/rust/operator-binary/src/extensions.rs +++ b/rust/operator-binary/src/extensions.rs @@ -4,7 +4,7 @@ use tracing::debug; use crate::{ authentication::DruidAuthenticationConfig, - crd::{security::DruidTlsSecurity, DbType, DruidCluster}, + crd::{security::DruidTlsSecurity, v1alpha1, DbType}, }; const EXT_S3: &str = "druid-s3-extensions"; @@ -20,7 +20,7 @@ const EXT_SIMPLE_CLIENT_SSL_CONTEXT: &str = "simple-client-sslcontext"; const ENV_PAC4J: &str = "druid-pac4j"; pub fn get_extension_list( - druid: &DruidCluster, + druid: &v1alpha1::DruidCluster, druid_tls_security: &DruidTlsSecurity, druid_auth_settings: &Option, ) -> Vec { @@ -82,8 +82,9 @@ mod tests { #[test] fn test_additional_extensions() { - let cluster = - deserialize_yaml_file::("test/resources/druid_controller/simple.yaml"); + let cluster = deserialize_yaml_file::( + "test/resources/druid_controller/simple.yaml", + ); assert_eq!( cluster.spec.cluster_config.additional_extensions, diff --git a/rust/operator-binary/src/internal_secret.rs b/rust/operator-binary/src/internal_secret.rs index 4535bb73..9691532b 100644 --- a/rust/operator-binary/src/internal_secret.rs +++ b/rust/operator-binary/src/internal_secret.rs @@ -9,9 +9,7 @@ use stackable_operator::{ }; use strum::{EnumDiscriminants, IntoStaticStr}; -use crate::crd::{ - security::INTERNAL_INITIAL_CLIENT_PASSWORD_ENV, DruidCluster, COOKIE_PASSPHRASE_ENV, -}; +use crate::crd::{security::INTERNAL_INITIAL_CLIENT_PASSWORD_ENV, v1alpha1, COOKIE_PASSPHRASE_ENV}; #[derive(Snafu, Debug, EnumDiscriminants)] #[strum_discriminants(derive(IntoStaticStr))] @@ -42,7 +40,7 @@ pub enum Error { } pub async fn create_shared_internal_secret( - druid: &DruidCluster, + druid: &v1alpha1::DruidCluster, client: &Client, controller_name: &str, ) -> Result<(), Error> { @@ -146,7 +144,7 @@ pub async fn create_shared_internal_secret( Ok(()) } -pub fn build_shared_internal_secret(druid: &DruidCluster) -> Result { +pub fn build_shared_internal_secret(druid: &v1alpha1::DruidCluster) -> Result { let mut internal_secret = BTreeMap::new(); internal_secret.insert( INTERNAL_INITIAL_CLIENT_PASSWORD_ENV.to_string(), @@ -166,11 +164,11 @@ pub fn build_shared_internal_secret(druid: &DruidCluster) -> Result String { +pub fn build_immutable_shared_internal_secret_name(druid: &v1alpha1::DruidCluster) -> String { format!("{}-internal-secret", druid.name_any()) } -pub fn build_shared_internal_secret_name(druid: &DruidCluster) -> String { +pub fn build_shared_internal_secret_name(druid: &v1alpha1::DruidCluster) -> String { format!("{}-shared-internal-secret", druid.name_any()) } diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index f9526483..7cdb0dbd 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -17,10 +17,11 @@ use stackable_operator::{ }, }, logging::controller::report_controller_reconciled, - CustomResourceExt, + shared::yaml::SerializeOptions, + YamlSchema, }; -use crate::crd::{DruidCluster, APP_NAME, OPERATOR_NAME}; +use crate::crd::{v1alpha1, DruidCluster, APP_NAME, OPERATOR_NAME}; mod authentication; mod config; @@ -47,7 +48,8 @@ struct Opts { async fn main() -> anyhow::Result<()> { let opts = Opts::parse(); match opts.cmd { - Command::Crd => DruidCluster::print_yaml_schema(built_info::PKG_VERSION)?, + Command::Crd => DruidCluster::merged_crd(DruidCluster::V1Alpha1)? + .print_yaml_schema(built_info::PKG_VERSION, SerializeOptions::default())?, Command::Run(ProductOperatorRun { product_config, watch_namespace, @@ -86,7 +88,7 @@ async fn main() -> anyhow::Result<()> { )); Controller::new( - watch_namespace.get_api::>(&client), + watch_namespace.get_api::>(&client), watcher::Config::default(), ) .owns( diff --git a/rust/operator-binary/src/operations/pdb.rs b/rust/operator-binary/src/operations/pdb.rs index 1c59fed7..f48692a0 100644 --- a/rust/operator-binary/src/operations/pdb.rs +++ b/rust/operator-binary/src/operations/pdb.rs @@ -5,7 +5,7 @@ use stackable_operator::{ }; use crate::{ - crd::{DruidCluster, DruidRole, APP_NAME, OPERATOR_NAME}, + crd::{v1alpha1, DruidRole, APP_NAME, OPERATOR_NAME}, druid_controller::DRUID_CONTROLLER_NAME, }; @@ -25,7 +25,7 @@ pub enum Error { pub async fn add_pdbs( pdb: &PdbConfig, - druid: &DruidCluster, + druid: &v1alpha1::DruidCluster, role: &DruidRole, client: &Client, cluster_resources: &mut ClusterResources, diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/product_logging.rs index fe9413f4..6f6f51b3 100644 --- a/rust/operator-binary/src/product_logging.rs +++ b/rust/operator-binary/src/product_logging.rs @@ -13,8 +13,7 @@ use stackable_operator::{ }; use crate::crd::{ - Container, DruidCluster, DRUID_LOG_FILE, LOG4J2_CONFIG, MAX_DRUID_LOG_FILES_SIZE, - STACKABLE_LOG_DIR, + v1alpha1, Container, DRUID_LOG_FILE, LOG4J2_CONFIG, MAX_DRUID_LOG_FILES_SIZE, STACKABLE_LOG_DIR, }; #[derive(Snafu, Debug)] @@ -43,7 +42,7 @@ const CONSOLE_CONVERSION_PATTERN: &str = "%d{ISO8601} %p [%t] %c - %m%n"; /// Return the address of the Vector aggregator if the corresponding ConfigMap name is given in the /// cluster spec pub async fn resolve_vector_aggregator_address( - druid: &DruidCluster, + druid: &v1alpha1::DruidCluster, client: &Client, ) -> Result> { let vector_aggregator_address = if let Some(vector_aggregator_config_map_name) = &druid @@ -80,7 +79,7 @@ pub async fn resolve_vector_aggregator_address( /// Extend the role group ConfigMap with logging and Vector configurations pub fn extend_role_group_config_map( - rolegroup: &RoleGroupRef, + rolegroup: &RoleGroupRef, vector_aggregator_address: Option<&str>, logging: &Logging, cm_builder: &mut ConfigMapBuilder, diff --git a/rustfmt.toml b/rustfmt.toml index 455c8209..709925ec 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -1,2 +1,4 @@ imports_granularity = "Crate" group_imports = "StdExternalCrate" +use_field_init_shorthand = true +reorder_impl_items = true From 34be263e537bf0b86c8aa236ca52a6febf50c331 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 31 Jan 2025 12:17:33 +0100 Subject: [PATCH 05/10] chore: Fix invalid references in doc comments --- rust/operator-binary/src/crd/mod.rs | 2 +- rust/operator-binary/src/discovery.rs | 4 ++-- rust/operator-binary/src/druid_controller.rs | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 98ce127e..3191bb2f 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -841,7 +841,7 @@ impl v1alpha1::DruidCluster { } /// Returns true if the cluster uses an s3 connection. - /// This is a quicker convenience function over the [DruidCluster::get_s3_connection] function. + /// This is a quicker convenience function over the [v1alpha1::DruidCluster::get_s3_connection] function. pub fn uses_s3(&self) -> bool { let s3_ingestion = self .spec diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index fcd6997c..5bc85895 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -37,7 +37,7 @@ pub enum Error { }, } -/// Builds discovery [`ConfigMap`]s for connecting to a [`DruidCluster`] +/// Builds discovery [`ConfigMap`]s for connecting to a [`v1alpha1::DruidCluster`]. pub async fn build_discovery_configmaps( druid: &v1alpha1::DruidCluster, owner: &impl Resource, @@ -56,7 +56,7 @@ pub async fn build_discovery_configmaps( )?]) } -/// Build a discovery [`ConfigMap`] containing information about how to connect to a certain [`DruidCluster`] +/// Build a discovery [`ConfigMap`] containing information about how to connect to a certain [`v1alpha1::DruidCluster`]. fn build_discovery_configmap( druid: &v1alpha1::DruidCluster, owner: &impl Resource, diff --git a/rust/operator-binary/src/druid_controller.rs b/rust/operator-binary/src/druid_controller.rs index 7d5d0f4b..0bd8628f 100644 --- a/rust/operator-binary/src/druid_controller.rs +++ b/rust/operator-binary/src/druid_controller.rs @@ -1,4 +1,6 @@ -//! Ensures that `Pod`s are configured and running for each [`DruidCluster`] +//! Ensures that `Pod`s are configured and running for each [`DruidCluster`][v1alpha1] +//! +//! [v1alpha1]: v1alpha1::DruidCluster use std::{ collections::{BTreeMap, HashMap}, str::FromStr, From 8e15732792fe12e115094c12bada8f85a1e1eb77 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 31 Jan 2025 11:17:22 +0100 Subject: [PATCH 06/10] chore: Remove separate CRD crate --- Cargo.lock | 19 ------- Cargo.toml | 7 +-- rust/crd/Cargo.toml | 25 --------- rust/operator-binary/Cargo.toml | 2 - .../src/authentication/ldap.rs | 11 ++-- .../operator-binary/src/authentication/mod.rs | 16 +++--- .../src/authentication/oidc.rs | 4 +- rust/operator-binary/src/config.rs | 5 +- .../src/crd}/affinity.rs | 12 ++--- .../src/crd}/authentication.rs | 13 ++--- .../src/crd}/authorization.rs | 0 .../src => operator-binary/src/crd}/memory.rs | 5 +- .../lib.rs => operator-binary/src/crd/mod.rs} | 39 +++++++------- .../src/crd}/resource.rs | 32 +++++------ .../src/crd}/security.rs | 12 ++--- .../src/crd}/storage.rs | 0 .../src => operator-binary/src/crd}/tls.rs | 3 +- rust/operator-binary/src/discovery.rs | 10 ++-- rust/operator-binary/src/druid_controller.rs | 54 +++++++++---------- rust/operator-binary/src/extensions.rs | 10 ++-- rust/operator-binary/src/internal_secret.rs | 16 +++--- rust/operator-binary/src/main.rs | 27 +++++----- .../src/operations/graceful_shutdown.rs | 6 +-- rust/operator-binary/src/operations/pdb.rs | 6 ++- rust/operator-binary/src/product_logging.rs | 9 ++-- .../crd}/resource_merge/druid_cluster.yaml | 0 .../crd}/resource_merge/segment_cache.yaml | 0 .../crd}/role_service/druid_cluster.yaml | 0 rustfmt.toml | 2 +- 29 files changed, 151 insertions(+), 194 deletions(-) delete mode 100644 rust/crd/Cargo.toml rename rust/{crd/src => operator-binary/src/crd}/affinity.rs (98%) rename rust/{crd/src => operator-binary/src/crd}/authentication.rs (99%) rename rust/{crd/src => operator-binary/src/crd}/authorization.rs (100%) rename rust/{crd/src => operator-binary/src/crd}/memory.rs (99%) rename rust/{crd/src/lib.rs => operator-binary/src/crd/mod.rs} (99%) rename rust/{crd/src => operator-binary/src/crd}/resource.rs (97%) rename rust/{crd/src => operator-binary/src/crd}/security.rs (99%) rename rust/{crd/src => operator-binary/src/crd}/storage.rs (100%) rename rust/{crd/src => operator-binary/src/crd}/tls.rs (98%) rename rust/{crd/test/resources => operator-binary/test/resources/crd}/resource_merge/druid_cluster.yaml (100%) rename rust/{crd/test/resources => operator-binary/test/resources/crd}/resource_merge/segment_cache.yaml (100%) rename rust/{crd/test/resources => operator-binary/test/resources/crd}/role_service/druid_cluster.yaml (100%) diff --git a/Cargo.lock b/Cargo.lock index 6a7e97f3..35277d06 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2382,24 +2382,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" -[[package]] -name = "stackable-druid-crd" -version = "0.0.0-dev" -dependencies = [ - "indoc", - "product-config", - "rstest", - "semver", - "serde", - "serde_json", - "serde_yaml", - "snafu 0.8.5", - "stackable-operator", - "strum", - "tokio", - "tracing", -] - [[package]] name = "stackable-druid-operator" version = "0.0.0-dev" @@ -2420,7 +2402,6 @@ dependencies = [ "serde_json", "serde_yaml", "snafu 0.8.5", - "stackable-druid-crd", "stackable-operator", "strum", "tokio", diff --git a/Cargo.toml b/Cargo.toml index b95d91a0..d48513df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["rust/crd", "rust/operator-binary"] +members = ["rust/operator-binary"] resolver = "2" [workspace.package] @@ -10,6 +10,9 @@ edition = "2021" repository = "https://github.com/stackabletech/druid-operator" [workspace.dependencies] +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.85.0" } +product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.7.0" } + anyhow = "1.0" built = { version = "0.7", features = ["chrono", "git2"] } clap = "4.5" @@ -18,7 +21,6 @@ fnv = "1.0" futures = { version = "0.3", features = ["compat"] } indoc = "2.0" openssl = "0.10" -product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.7.0" } pin-project = "1.1" rstest = "0.24" semver = "1.0" @@ -26,7 +28,6 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_yaml = "0.9" snafu = "0.8" -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.85.0" } strum = { version = "0.26", features = ["derive"] } tokio = { version = "1.40", features = ["full"] } tracing = "0.1" diff --git a/rust/crd/Cargo.toml b/rust/crd/Cargo.toml deleted file mode 100644 index 2e2aa3aa..00000000 --- a/rust/crd/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -name = "stackable-druid-crd" -description = "Contains the Apache Druid CRD structs and utilities" -version.workspace = true -authors.workspace = true -license.workspace = true -edition.workspace = true -repository.workspace = true -publish = false - -[dependencies] -indoc.workspace = true -product-config.workspace = true -semver.workspace = true -serde.workspace = true -serde_json.workspace = true -stackable-operator.workspace = true -strum.workspace = true -tracing.workspace = true -snafu.workspace = true - -[dev-dependencies] -rstest.workspace = true -serde_yaml.workspace = true -tokio.workspace = true diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index e282d4ca..5eaa4564 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -9,8 +9,6 @@ repository.workspace = true publish = false [dependencies] -stackable-druid-crd = { path = "../crd" } - anyhow.workspace = true clap.workspace = true const_format.workspace = true diff --git a/rust/operator-binary/src/authentication/ldap.rs b/rust/operator-binary/src/authentication/ldap.rs index ac8e37d9..bfed3c82 100644 --- a/rust/operator-binary/src/authentication/ldap.rs +++ b/rust/operator-binary/src/authentication/ldap.rs @@ -6,12 +6,11 @@ use stackable_operator::{ commons::authentication::ldap::AuthenticationProvider, }; -use stackable_druid_crd::security::{ - add_cert_to_trust_store_cmd, STACKABLE_TLS_DIR, TLS_STORE_PASSWORD, -}; - -use crate::authentication::{ - AddLdapVolumesSnafu, ConstructLdapEndpointUrlSnafu, Error, MissingLdapBindCredentialsSnafu, +use crate::{ + authentication::{ + AddLdapVolumesSnafu, ConstructLdapEndpointUrlSnafu, Error, MissingLdapBindCredentialsSnafu, + }, + crd::security::{add_cert_to_trust_store_cmd, STACKABLE_TLS_DIR, TLS_STORE_PASSWORD}, }; fn add_authenticator_config( diff --git a/rust/operator-binary/src/authentication/mod.rs b/rust/operator-binary/src/authentication/mod.rs index 75c67d07..8a92b69a 100644 --- a/rust/operator-binary/src/authentication/mod.rs +++ b/rust/operator-binary/src/authentication/mod.rs @@ -1,11 +1,6 @@ use std::collections::BTreeMap; use snafu::Snafu; -use stackable_druid_crd::{ - authentication::{AuthenticationClassResolved, AuthenticationClassesResolved}, - security::{ESCALATOR_INTERNAL_CLIENT_PASSWORD_ENV, INTERNAL_INITIAL_CLIENT_PASSWORD_ENV}, - DruidCluster, DruidRole, -}; use stackable_operator::{ builder::pod::{container::ContainerBuilder, PodBuilder}, commons::{ @@ -20,11 +15,18 @@ use stackable_operator::{ k8s_openapi::api::core::v1::EnvVar, }; +use crate::{ + crd::{ + authentication::{AuthenticationClassResolved, AuthenticationClassesResolved}, + security::{ESCALATOR_INTERNAL_CLIENT_PASSWORD_ENV, INTERNAL_INITIAL_CLIENT_PASSWORD_ENV}, + DruidCluster, DruidRole, + }, + internal_secret::{build_shared_internal_secret_name, env_var_from_secret}, +}; + pub mod ldap; pub mod oidc; -use crate::internal_secret::{build_shared_internal_secret_name, env_var_from_secret}; - type Result = std::result::Result; #[derive(Snafu, Debug)] diff --git a/rust/operator-binary/src/authentication/oidc.rs b/rust/operator-binary/src/authentication/oidc.rs index 8fb65270..13ee72e4 100644 --- a/rust/operator-binary/src/authentication/oidc.rs +++ b/rust/operator-binary/src/authentication/oidc.rs @@ -1,9 +1,6 @@ use std::collections::BTreeMap; use snafu::ResultExt; -use stackable_druid_crd::{ - security::add_cert_to_jvm_trust_store_cmd, DruidRole, COOKIE_PASSPHRASE_ENV, -}; use stackable_operator::{ builder::pod::{container::ContainerBuilder, PodBuilder}, commons::authentication::oidc::{AuthenticationProvider, ClientAuthenticationOptions}, @@ -12,6 +9,7 @@ use stackable_operator::{ use crate::{ authentication::{AddOidcVolumesSnafu, ConstructOidcWellKnownUrlSnafu, Error}, + crd::{security::add_cert_to_jvm_trust_store_cmd, DruidRole, COOKIE_PASSPHRASE_ENV}, internal_secret::env_var_from_secret, }; diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs index 908718e1..8f3af715 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config.rs @@ -1,10 +1,11 @@ use indoc::formatdoc; use snafu::{ResultExt, Snafu}; -use stackable_druid_crd::{ +use stackable_operator::memory::MemoryQuantity; + +use crate::crd::{ DruidRole, JVM_SECURITY_PROPERTIES_FILE, LOG4J2_CONFIG, RW_CONFIG_DIRECTORY, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD, }; -use stackable_operator::memory::MemoryQuantity; #[derive(Snafu, Debug)] pub enum Error { diff --git a/rust/crd/src/affinity.rs b/rust/operator-binary/src/crd/affinity.rs similarity index 98% rename from rust/crd/src/affinity.rs rename to rust/operator-binary/src/crd/affinity.rs index 275d10bd..6f0b3b9c 100644 --- a/rust/crd/src/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -3,7 +3,7 @@ use stackable_operator::{ k8s_openapi::api::core::v1::{PodAffinity, PodAntiAffinity}, }; -use crate::{DeepStorageSpec, DruidRole, HdfsDeepStorageSpec, APP_NAME}; +use crate::crd::{DeepStorageSpec, DruidRole, HdfsDeepStorageSpec, APP_NAME}; /// Please have a look at the architecture diagram in /// to understand which roles do communicate with each other. @@ -32,7 +32,7 @@ pub fn get_affinity( DruidRole::Historical | // Ingests data DruidRole::MiddleManager => { - if let DeepStorageSpec::HDFS(HdfsDeepStorageSpec { + if let DeepStorageSpec::Hdfs(HdfsDeepStorageSpec { config_map_name: hdfs_discovery_cm_name, .. }) = deep_storage @@ -76,12 +76,9 @@ pub fn get_affinity( #[cfg(test)] mod tests { - use super::*; - - use rstest::rstest; use std::collections::BTreeMap; - use crate::DruidCluster; + use rstest::rstest; use stackable_operator::{ commons::affinity::StackableAffinity, k8s_openapi::{ @@ -92,6 +89,9 @@ mod tests { }, }; + use super::*; + use crate::crd::DruidCluster; + #[rstest] #[case(DruidRole::Coordinator)] #[case(DruidRole::Broker)] diff --git a/rust/crd/src/authentication.rs b/rust/operator-binary/src/crd/authentication.rs similarity index 99% rename from rust/crd/src/authentication.rs rename to rust/operator-binary/src/crd/authentication.rs index 537a3b22..65735a18 100644 --- a/rust/crd/src/authentication.rs +++ b/rust/operator-binary/src/crd/authentication.rs @@ -12,7 +12,7 @@ use stackable_operator::{ }; use tracing::info; -use crate::DruidClusterConfig; +use crate::crd::DruidClusterConfig; type Result = std::result::Result; @@ -232,19 +232,14 @@ impl AuthenticationClassesResolved { #[cfg(test)] mod tests { - use indoc::formatdoc; - use oidc::ClientAuthenticationOptions; - - use crate::DruidClusterConfig; - use std::pin::Pin; - use indoc::indoc; + use indoc::{formatdoc, indoc}; + use oidc::ClientAuthenticationOptions; use stackable_operator::kube; use super::*; - - use crate::authentication::AuthenticationClassesResolved; + use crate::crd::{authentication::AuthenticationClassesResolved, DruidClusterConfig}; const BASE_CLUSTER_CONFIG: &str = r#" deepStorage: diff --git a/rust/crd/src/authorization.rs b/rust/operator-binary/src/crd/authorization.rs similarity index 100% rename from rust/crd/src/authorization.rs rename to rust/operator-binary/src/crd/authorization.rs diff --git a/rust/crd/src/memory.rs b/rust/operator-binary/src/crd/memory.rs similarity index 99% rename from rust/crd/src/memory.rs rename to rust/operator-binary/src/crd/memory.rs index b4fb005c..c3da9034 100644 --- a/rust/crd/src/memory.rs +++ b/rust/operator-binary/src/crd/memory.rs @@ -7,7 +7,7 @@ use stackable_operator::{ memory::{BinaryMultiple, MemoryQuantity}, }; -use crate::{ +use crate::crd::{ storage::HistoricalStorage, PROCESSING_BUFFER_SIZE_BYTES, PROCESSING_NUM_MERGE_BUFFERS, PROCESSING_NUM_THREADS, }; @@ -167,9 +167,10 @@ fn format_for_druid(memory_quantity: &MemoryQuantity) -> String { #[cfg(test)] mod tests { - use super::*; use rstest::*; + use super::*; + #[rstest] #[case(1000, 1)] #[case(1400, 1)] diff --git a/rust/crd/src/lib.rs b/rust/operator-binary/src/crd/mod.rs similarity index 99% rename from rust/crd/src/lib.rs rename to rust/operator-binary/src/crd/mod.rs index 84a2be00..3addd72f 100644 --- a/rust/crd/src/lib.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,18 +1,4 @@ -pub mod affinity; -pub mod authentication; -pub mod authorization; -pub mod memory; -pub mod resource; -pub mod security; -pub mod storage; -pub mod tls; - -use crate::{ - affinity::get_affinity, - authorization::DruidAuthorization, - resource::RoleResource, - tls::{default_druid_tls, DruidTls}, -}; +use std::collections::{BTreeMap, HashMap, HashSet}; use indoc::formatdoc; use product_config::types::PropertyNameKind; @@ -55,9 +41,24 @@ use stackable_operator::{ COMMON_BASH_TRAP_FUNCTIONS, }, }; -use std::collections::{BTreeMap, HashMap, HashSet}; use strum::{Display, EnumDiscriminants, EnumIter, EnumString, IntoStaticStr}; +use crate::crd::{ + affinity::get_affinity, + authorization::DruidAuthorization, + resource::RoleResource, + tls::{default_druid_tls, DruidTls}, +}; + +pub mod affinity; +pub mod authentication; +pub mod authorization; +pub mod memory; +pub mod resource; +pub mod security; +pub mod storage; +pub mod tls; + pub const APP_NAME: &str = "druid"; pub const OPERATOR_NAME: &str = "druid.stackable.tech"; @@ -690,7 +691,7 @@ impl DruidCluster { Some(self.spec.cluster_config.deep_storage.to_string()), ); match self.spec.cluster_config.deep_storage.clone() { - DeepStorageSpec::HDFS(hdfs) => { + DeepStorageSpec::Hdfs(hdfs) => { result.insert(DS_DIRECTORY.to_string(), Some(hdfs.directory)); } DeepStorageSpec::S3(s3_spec) => { @@ -1076,7 +1077,7 @@ pub enum DeepStorageSpec { /// You can run an HDFS cluster with the [Stackable operator for Apache HDFS](DOCS_BASE_URL_PLACEHOLDER/hdfs/). #[serde(rename = "hdfs")] #[strum(serialize = "hdfs")] - HDFS(HdfsDeepStorageSpec), + Hdfs(HdfsDeepStorageSpec), /// [The S3 deep storage configuration](DOCS_BASE_URL_PLACEHOLDER/druid/usage-guide/deep-storage#_s3). #[strum(serialize = "s3")] S3(S3DeepStorageSpec), @@ -1084,7 +1085,7 @@ pub enum DeepStorageSpec { impl DeepStorageSpec { pub fn is_hdfs(&self) -> bool { - matches!(self, DeepStorageSpec::HDFS(_)) + matches!(self, DeepStorageSpec::Hdfs(_)) } pub fn is_s3(&self) -> bool { matches!(self, DeepStorageSpec::S3(_)) diff --git a/rust/crd/src/resource.rs b/rust/operator-binary/src/crd/resource.rs similarity index 97% rename from rust/crd/src/resource.rs rename to rust/operator-binary/src/crd/resource.rs index f122a566..9a53f019 100644 --- a/rust/crd/src/resource.rs +++ b/rust/operator-binary/src/crd/resource.rs @@ -1,13 +1,8 @@ -use std::collections::BTreeMap; -use std::sync::LazyLock; +use std::{collections::BTreeMap, sync::LazyLock}; -use crate::memory::{HistoricalDerivedSettings, RESERVED_OS_MEMORY}; -use crate::storage::{self, default_free_percentage_empty_dir_fragment}; -use crate::{DruidRole, PATH_SEGMENT_CACHE, PROP_SEGMENT_CACHE_LOCATIONS}; use snafu::{OptionExt, ResultExt, Snafu}; -use stackable_operator::builder; -use stackable_operator::memory::MemoryQuantity; use stackable_operator::{ + builder, builder::pod::{container::ContainerBuilder, volume::VolumeBuilder, PodBuilder}, commons::resources::{ CpuLimitsFragment, MemoryLimits, MemoryLimitsFragment, NoRuntimeLimits, @@ -17,9 +12,16 @@ use stackable_operator::{ api::core::v1::{EmptyDirVolumeSource, ResourceRequirements}, apimachinery::pkg::api::resource::Quantity, }, + memory::MemoryQuantity, }; use strum::{EnumDiscriminants, IntoStaticStr}; +use crate::crd::{ + memory::{HistoricalDerivedSettings, RESERVED_OS_MEMORY}, + storage::{self, default_free_percentage_empty_dir_fragment}, + DruidRole, PATH_SEGMENT_CACHE, PROP_SEGMENT_CACHE_LOCATIONS, +}; + // volume names const SEGMENT_CACHE_VOLUME_NAME: &str = "segment-cache"; @@ -29,7 +31,7 @@ const SEGMENT_CACHE_VOLUME_NAME: &str = "segment-cache"; #[allow(clippy::enum_variant_names)] pub enum Error { #[snafu(display("failed to derive Druid settings from resources"))] - DeriveMemorySettings { source: crate::memory::Error }, + DeriveMemorySettings { source: crate::crd::memory::Error }, #[snafu(display("failed to get memory limits"))] GetMemoryLimit, @@ -242,13 +244,6 @@ pub static ROUTER_RESOURCES: LazyLock Option { #[cfg(test)] mod tests { - use crate::{tests::deserialize_yaml_str, tls::DruidTls, DruidClusterConfig}; use indoc::formatdoc; + use crate::crd::{tests::deserialize_yaml_str, tls::DruidTls, DruidClusterConfig}; + const BASE_DRUID_CONFIGURATION: &str = r#" deepStorage: hdfs: diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index 0100208b..2c6b9b31 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -1,12 +1,7 @@ //! Discovery for Druid. We make Druid discoverable by putting a connection string to the router service //! inside a config map. We only provide a connection string to the router service, since it serves as //! a gateway to the cluster for client queries. -use crate::DRUID_CONTROLLER_NAME; - use snafu::{OptionExt, ResultExt, Snafu}; -use stackable_druid_crd::{ - build_recommended_labels, security::DruidTlsSecurity, DruidCluster, DruidRole, -}; use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, commons::product_image_selection::ResolvedProductImage, @@ -15,6 +10,11 @@ use stackable_operator::{ utils::cluster_info::KubernetesClusterInfo, }; +use crate::{ + crd::{build_recommended_labels, security::DruidTlsSecurity, DruidCluster, DruidRole}, + DRUID_CONTROLLER_NAME, +}; + #[derive(Snafu, Debug)] pub enum Error { #[snafu(display("object {} is missing metadata to build owner reference", druid))] diff --git a/rust/operator-binary/src/druid_controller.rs b/rust/operator-binary/src/druid_controller.rs index 0715680b..64b3bfe2 100644 --- a/rust/operator-binary/src/druid_controller.rs +++ b/rust/operator-binary/src/druid_controller.rs @@ -12,16 +12,6 @@ use product_config::{ ProductConfigManager, }; use snafu::{OptionExt, ResultExt, Snafu}; -use stackable_druid_crd::{ - authentication::AuthenticationClassesResolved, authorization::DruidAuthorization, - build_recommended_labels, build_string_list, security::DruidTlsSecurity, CommonRoleGroupConfig, - Container, DeepStorageSpec, DruidCluster, DruidClusterStatus, DruidRole, APP_NAME, - AUTH_AUTHORIZER_OPA_URI, CREDENTIALS_SECRET_PROPERTY, DB_PASSWORD_ENV, DB_USERNAME_ENV, - DRUID_CONFIG_DIRECTORY, DS_BUCKET, EXTENSIONS_LOADLIST, HDFS_CONFIG_DIRECTORY, JVM_CONFIG, - JVM_SECURITY_PROPERTIES_FILE, LOG_CONFIG_DIRECTORY, MAX_DRUID_LOG_FILES_SIZE, RUNTIME_PROPS, - RW_CONFIG_DIRECTORY, S3_ACCESS_KEY, S3_ENDPOINT_URL, S3_PATH_STYLE_ACCESS, S3_SECRET_KEY, - STACKABLE_LOG_DIR, ZOOKEEPER_CONNECTION_STRING, -}; use stackable_operator::{ builder::{ self, @@ -76,12 +66,22 @@ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ authentication::DruidAuthenticationConfig, config::get_jvm_config, + crd::{ + authentication::AuthenticationClassesResolved, authorization::DruidAuthorization, + build_recommended_labels, build_string_list, security::DruidTlsSecurity, + CommonRoleGroupConfig, Container, DeepStorageSpec, DruidCluster, DruidClusterStatus, + DruidRole, APP_NAME, AUTH_AUTHORIZER_OPA_URI, CREDENTIALS_SECRET_PROPERTY, DB_PASSWORD_ENV, + DB_USERNAME_ENV, DRUID_CONFIG_DIRECTORY, DS_BUCKET, EXTENSIONS_LOADLIST, + HDFS_CONFIG_DIRECTORY, JVM_CONFIG, JVM_SECURITY_PROPERTIES_FILE, LOG_CONFIG_DIRECTORY, + MAX_DRUID_LOG_FILES_SIZE, OPERATOR_NAME, RUNTIME_PROPS, RW_CONFIG_DIRECTORY, S3_ACCESS_KEY, + S3_ENDPOINT_URL, S3_PATH_STYLE_ACCESS, S3_SECRET_KEY, STACKABLE_LOG_DIR, + ZOOKEEPER_CONNECTION_STRING, + }, discovery::{self, build_discovery_configmaps}, extensions::get_extension_list, internal_secret::{create_shared_internal_secret, env_var_from_secret}, operations::{graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs}, product_logging::{extend_role_group_config_map, resolve_vector_aggregator_address}, - OPERATOR_NAME, }; pub const DRUID_CONTROLLER_NAME: &str = "druidcluster"; @@ -170,7 +170,7 @@ pub enum Error { }, #[snafu(display("failed to get valid S3 connection"))] - GetS3Connection { source: stackable_druid_crd::Error }, + GetS3Connection { source: crate::crd::Error }, #[snafu(display("failed to configure S3 connection"))] ConfigureS3 { source: S3Error }, @@ -220,10 +220,10 @@ pub enum Error { }, #[snafu(display("failed to resolve and merge config for role and role group"))] - FailedToResolveConfig { source: stackable_druid_crd::Error }, + FailedToResolveConfig { source: crate::crd::Error }, #[snafu(display("invalid configuration"))] - InvalidConfiguration { source: stackable_druid_crd::Error }, + InvalidConfiguration { source: crate::crd::Error }, #[snafu(display("failed to create cluster resources"))] CreateClusterResources { @@ -245,27 +245,21 @@ pub enum Error { ObjectHasNoNamespace, #[snafu(display("failed to initialize security context"))] - FailedToInitializeSecurityContext { - source: stackable_druid_crd::security::Error, - }, + FailedToInitializeSecurityContext { source: crate::crd::security::Error }, #[snafu(display("failed to retrieve AuthenticationClass"))] AuthenticationClassRetrieval { - source: stackable_druid_crd::authentication::Error, + source: crate::crd::authentication::Error, }, #[snafu(display("failed to get JVM config"))] GetJvmConfig { source: crate::config::Error }, #[snafu(display("failed to derive Druid memory settings from resources"))] - DeriveMemorySettings { - source: stackable_druid_crd::resource::Error, - }, + DeriveMemorySettings { source: crate::crd::resource::Error }, #[snafu(display("failed to update Druid config from resources"))] - UpdateDruidConfigFromResources { - source: stackable_druid_crd::resource::Error, - }, + UpdateDruidConfigFromResources { source: crate::crd::resource::Error }, #[snafu(display("failed to retrieve secret for internal communications"))] FailedInternalSecretCreation { @@ -1218,7 +1212,7 @@ fn add_hdfs_cm_volume_and_volume_mounts( pb: &mut PodBuilder, ) -> Result<()> { // hdfs deep storage mount - if let DeepStorageSpec::HDFS(hdfs) = deep_storage_spec { + if let DeepStorageSpec::Hdfs(hdfs) = deep_storage_spec { cb_druid .add_volume_mount(HDFS_CONFIG_VOLUME_NAME, HDFS_CONFIG_DIRECTORY) .context(AddVolumeMountSnafu)?; @@ -1334,11 +1328,11 @@ pub fn error_policy( #[cfg(test)] mod test { - use super::*; - use product_config::{writer, ProductConfigManager}; use rstest::*; - use stackable_druid_crd::PROP_SEGMENT_CACHE_LOCATIONS; + + use super::*; + use crate::crd::PROP_SEGMENT_CACHE_LOCATIONS; #[derive(Snafu, Debug, EnumDiscriminants)] #[strum_discriminants(derive(IntoStaticStr))] @@ -1359,9 +1353,9 @@ mod test { source: stackable_operator::product_config_utils::Error, }, #[snafu(display("failed to resolve and merge config for role and role group"))] - FailedToResolveConfig { source: stackable_druid_crd::Error }, + FailedToResolveConfig { source: crate::crd::Error }, #[snafu(display("invalid configuration"))] - InvalidConfiguration { source: stackable_druid_crd::Error }, + InvalidConfiguration { source: crate::crd::Error }, } #[rstest] diff --git a/rust/operator-binary/src/extensions.rs b/rust/operator-binary/src/extensions.rs index 847ffc6f..5212211b 100644 --- a/rust/operator-binary/src/extensions.rs +++ b/rust/operator-binary/src/extensions.rs @@ -1,9 +1,11 @@ use std::collections::HashSet; -use stackable_druid_crd::{security::DruidTlsSecurity, DbType, DruidCluster}; use tracing::debug; -use crate::authentication::DruidAuthenticationConfig; +use crate::{ + authentication::DruidAuthenticationConfig, + crd::{security::DruidTlsSecurity, DbType, DruidCluster}, +}; const EXT_S3: &str = "druid-s3-extensions"; const EXT_KAFKA_INDEXING: &str = "druid-kafka-indexing-service"; @@ -70,15 +72,13 @@ pub fn get_extension_list( #[cfg(test)] mod tests { - use stackable_druid_crd::authentication::{ - AuthenticationClassResolved, AuthenticationClassesResolved, - }; use stackable_operator::commons::{ authentication::oidc::{AuthenticationProvider, ClientAuthenticationOptions}, tls_verification::TlsClientDetails, }; use super::*; + use crate::crd::authentication::{AuthenticationClassResolved, AuthenticationClassesResolved}; #[test] fn test_additional_extensions() { diff --git a/rust/operator-binary/src/internal_secret.rs b/rust/operator-binary/src/internal_secret.rs index 677b5d41..4535bb73 100644 --- a/rust/operator-binary/src/internal_secret.rs +++ b/rust/operator-binary/src/internal_secret.rs @@ -1,14 +1,18 @@ +use std::collections::{BTreeMap, HashSet}; + use snafu::{OptionExt, ResultExt, Snafu}; -use stackable_druid_crd::security::INTERNAL_INITIAL_CLIENT_PASSWORD_ENV; -use stackable_druid_crd::{DruidCluster, COOKIE_PASSPHRASE_ENV}; -use stackable_operator::k8s_openapi::api::core::v1::{EnvVar, EnvVarSource, SecretKeySelector}; -use stackable_operator::kube::ResourceExt; use stackable_operator::{ - builder::meta::ObjectMetaBuilder, client::Client, k8s_openapi::api::core::v1::Secret, + builder::meta::ObjectMetaBuilder, + client::Client, + k8s_openapi::api::core::v1::{EnvVar, EnvVarSource, Secret, SecretKeySelector}, + kube::ResourceExt, }; -use std::collections::{BTreeMap, HashSet}; use strum::{EnumDiscriminants, IntoStaticStr}; +use crate::crd::{ + security::INTERNAL_INITIAL_CLIENT_PASSWORD_ENV, DruidCluster, COOKIE_PASSPHRASE_ENV, +}; + #[derive(Snafu, Debug, EnumDiscriminants)] #[strum_discriminants(derive(IntoStaticStr))] #[allow(clippy::enum_variant_names)] diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index b22a760c..f9526483 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -1,22 +1,10 @@ -mod authentication; -mod config; -mod discovery; -mod druid_controller; -mod extensions; -mod internal_secret; -mod operations; -mod product_logging; - use std::sync::Arc; use clap::{crate_description, crate_version, Parser}; use druid_controller::{DRUID_CONTROLLER_NAME, FULL_CONTROLLER_NAME}; use futures::StreamExt; -use stackable_druid_crd::{DruidCluster, APP_NAME, OPERATOR_NAME}; -use stackable_operator::CustomResourceExt; use stackable_operator::{ - cli::Command, - cli::ProductOperatorRun, + cli::{Command, ProductOperatorRun}, k8s_openapi::api::{ apps::v1::StatefulSet, core::v1::{ConfigMap, Service}, @@ -29,8 +17,21 @@ use stackable_operator::{ }, }, logging::controller::report_controller_reconciled, + CustomResourceExt, }; +use crate::crd::{DruidCluster, APP_NAME, OPERATOR_NAME}; + +mod authentication; +mod config; +mod crd; +mod discovery; +mod druid_controller; +mod extensions; +mod internal_secret; +mod operations; +mod product_logging; + mod built_info { include!(concat!(env!("OUT_DIR"), "/built.rs")); } diff --git a/rust/operator-binary/src/operations/graceful_shutdown.rs b/rust/operator-binary/src/operations/graceful_shutdown.rs index eabd1e8f..fe01ad8d 100644 --- a/rust/operator-binary/src/operations/graceful_shutdown.rs +++ b/rust/operator-binary/src/operations/graceful_shutdown.rs @@ -1,13 +1,13 @@ use indoc::formatdoc; use snafu::{ResultExt, Snafu}; -use stackable_druid_crd::security::DruidTlsSecurity; -use stackable_druid_crd::DruidRole; -use stackable_operator::k8s_openapi::api::core::v1::{ExecAction, LifecycleHandler}; use stackable_operator::{ builder::pod::{container::ContainerBuilder, PodBuilder}, + k8s_openapi::api::core::v1::{ExecAction, LifecycleHandler}, time::Duration, }; +use crate::crd::{security::DruidTlsSecurity, DruidRole}; + #[derive(Debug, Snafu)] pub enum Error { #[snafu(display("Failed to set terminationGracePeriod"))] diff --git a/rust/operator-binary/src/operations/pdb.rs b/rust/operator-binary/src/operations/pdb.rs index 850e4723..1c59fed7 100644 --- a/rust/operator-binary/src/operations/pdb.rs +++ b/rust/operator-binary/src/operations/pdb.rs @@ -1,11 +1,13 @@ use snafu::{ResultExt, Snafu}; -use stackable_druid_crd::{DruidCluster, DruidRole, APP_NAME, OPERATOR_NAME}; use stackable_operator::{ builder::pdb::PodDisruptionBudgetBuilder, client::Client, cluster_resources::ClusterResources, commons::pdb::PdbConfig, kube::ResourceExt, }; -use crate::druid_controller::DRUID_CONTROLLER_NAME; +use crate::{ + crd::{DruidCluster, DruidRole, APP_NAME, OPERATOR_NAME}, + druid_controller::DRUID_CONTROLLER_NAME, +}; #[derive(Snafu, Debug)] pub enum Error { diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/product_logging.rs index 22512884..fe9413f4 100644 --- a/rust/operator-binary/src/product_logging.rs +++ b/rust/operator-binary/src/product_logging.rs @@ -1,8 +1,4 @@ use snafu::{OptionExt, ResultExt, Snafu}; -use stackable_druid_crd::{ - Container, DruidCluster, DRUID_LOG_FILE, LOG4J2_CONFIG, MAX_DRUID_LOG_FILES_SIZE, - STACKABLE_LOG_DIR, -}; use stackable_operator::{ builder::configmap::ConfigMapBuilder, client::Client, @@ -16,6 +12,11 @@ use stackable_operator::{ role_utils::RoleGroupRef, }; +use crate::crd::{ + Container, DruidCluster, DRUID_LOG_FILE, LOG4J2_CONFIG, MAX_DRUID_LOG_FILES_SIZE, + STACKABLE_LOG_DIR, +}; + #[derive(Snafu, Debug)] pub enum Error { #[snafu(display("object has no namespace"))] diff --git a/rust/crd/test/resources/resource_merge/druid_cluster.yaml b/rust/operator-binary/test/resources/crd/resource_merge/druid_cluster.yaml similarity index 100% rename from rust/crd/test/resources/resource_merge/druid_cluster.yaml rename to rust/operator-binary/test/resources/crd/resource_merge/druid_cluster.yaml diff --git a/rust/crd/test/resources/resource_merge/segment_cache.yaml b/rust/operator-binary/test/resources/crd/resource_merge/segment_cache.yaml similarity index 100% rename from rust/crd/test/resources/resource_merge/segment_cache.yaml rename to rust/operator-binary/test/resources/crd/resource_merge/segment_cache.yaml diff --git a/rust/crd/test/resources/role_service/druid_cluster.yaml b/rust/operator-binary/test/resources/crd/role_service/druid_cluster.yaml similarity index 100% rename from rust/crd/test/resources/role_service/druid_cluster.yaml rename to rust/operator-binary/test/resources/crd/role_service/druid_cluster.yaml diff --git a/rustfmt.toml b/rustfmt.toml index e25d3967..6f13bf68 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -2,4 +2,4 @@ # It's also ok to use the stable toolchain by simple running "cargo fmt", but using the nigthly formatter is prefered. imports_granularity = "Crate" -group_imports = "StdExternalCrate" +group_imports = "StdExternalCrate" From e6c5e7a847fa26f45be2038fc447587800abd294 Mon Sep 17 00:00:00 2001 From: Techassi Date: Tue, 4 Feb 2025 10:05:44 +0100 Subject: [PATCH 07/10] chore: Bump openssl to 0.10.70 to fix RUSTSEC-2025-0004 --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 97c2a10a..cd91eb95 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1778,9 +1778,9 @@ checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "openssl" -version = "0.10.68" +version = "0.10.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" +checksum = "61cfb4e166a8bb8c9b55c500bc2308550148ece889be90f609377e58140f42c6" dependencies = [ "bitflags 2.8.0", "cfg-if", @@ -1810,9 +1810,9 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "openssl-sys" -version = "0.9.104" +version = "0.9.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" +checksum = "8b22d5b84be05a8d6947c7cb71f7c849aa0f112acd4bf51c2a7c1c988ac0a9dc" dependencies = [ "cc", "libc", From 536a8c7899b33e83c2446a6af27db455e675b593 Mon Sep 17 00:00:00 2001 From: Techassi Date: Tue, 4 Feb 2025 13:25:26 +0100 Subject: [PATCH 08/10] chore: Version DruidClusterConfig --- .../operator-binary/src/crd/authentication.rs | 4 +- rust/operator-binary/src/crd/mod.rs | 154 +++++++++--------- rust/operator-binary/src/crd/tls.rs | 2 +- 3 files changed, 80 insertions(+), 80 deletions(-) diff --git a/rust/operator-binary/src/crd/authentication.rs b/rust/operator-binary/src/crd/authentication.rs index c86dde3f..364de277 100644 --- a/rust/operator-binary/src/crd/authentication.rs +++ b/rust/operator-binary/src/crd/authentication.rs @@ -12,7 +12,7 @@ use stackable_operator::{ }; use tracing::info; -use crate::crd::DruidClusterConfig; +use crate::crd::v1alpha1::DruidClusterConfig; type Result = std::result::Result; @@ -230,7 +230,7 @@ mod tests { use stackable_operator::kube; use super::*; - use crate::crd::{authentication::AuthenticationClassesResolved, DruidClusterConfig}; + use crate::crd::{authentication::AuthenticationClassesResolved, v1alpha1::DruidClusterConfig}; const BASE_CLUSTER_CONFIG: &str = r#" deepStorage: diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 3191bb2f..9578e0f7 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -202,7 +202,7 @@ pub mod versioned { #[serde(rename_all = "camelCase")] pub struct DruidClusterSpec { /// Common cluster wide configuration that can not differ or be overridden on a role or role group level. - pub cluster_config: DruidClusterConfig, + pub cluster_config: v1alpha1::DruidClusterConfig, // no doc - docs provided by the struct. pub image: ProductImage, @@ -226,6 +226,82 @@ pub mod versioned { #[serde(default)] pub cluster_operation: ClusterOperation, } + + #[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct DruidClusterConfig { + /// Additional extensions to load in Druid. + /// The operator will automatically load all extensions needed based on the cluster + /// configuration, but for extra functionality which the operator cannot anticipate, it can + /// sometimes be necessary to load additional extensions. + /// Add configuration for additional extensions using [configuration override for Druid](https://docs.stackable.tech/home/stable/druid/usage-guide/configuration-and-environment-overrides). + #[serde(default)] + pub additional_extensions: HashSet, + + /// List of [AuthenticationClasses](DOCS_BASE_URL_PLACEHOLDER/concepts/authentication) + /// to use for authenticating users. TLS, LDAP and OIDC authentication are supported. More information in + /// the [Druid operator security documentation](DOCS_BASE_URL_PLACEHOLDER/druid/usage-guide/security#_authentication). + /// + /// For TLS: Please note that the SecretClass used to authenticate users needs to be the same + /// as the SecretClass used for internal communication. + #[serde(default)] + pub authentication: Vec, + + /// Authorization settings for Druid like OPA + #[serde(skip_serializing_if = "Option::is_none")] + pub authorization: Option, + + /// [Druid deep storage configuration](DOCS_BASE_URL_PLACEHOLDER/druid/usage-guide/deep-storage). + /// Only one backend can be used at a time. Either HDFS or S3 are supported. + pub deep_storage: DeepStorageSpec, + + /// Configuration properties for data ingestion tasks. + #[serde(skip_serializing_if = "Option::is_none")] + pub ingestion: Option, + + /// Druid requires an SQL database to store metadata into. Specify connection information here. + pub metadata_storage_database: DatabaseConnectionSpec, + + /// TLS encryption settings for Druid, more information in the + /// [security documentation](DOCS_BASE_URL_PLACEHOLDER/druid/usage-guide/security). + /// This setting only affects server and internal communication. + /// It does not affect client tls authentication, use `clusterConfig.authentication` instead. + #[serde(default = "default_druid_tls", skip_serializing_if = "Option::is_none")] + pub tls: Option, + + /// Druid requires a ZooKeeper cluster connection to run. + /// Provide the name of the ZooKeeper [discovery ConfigMap](DOCS_BASE_URL_PLACEHOLDER/concepts/service_discovery) + /// here. When using the [Stackable operator for Apache ZooKeeper](DOCS_BASE_URL_PLACEHOLDER/zookeeper/) + /// to deploy a ZooKeeper cluster, this will simply be the name of your ZookeeperCluster resource. + pub zookeeper_config_map_name: String, + + /// Name of the Vector aggregator [discovery ConfigMap](DOCS_BASE_URL_PLACEHOLDER/concepts/service_discovery). + /// It must contain the key `ADDRESS` with the address of the Vector aggregator. + /// Follow the [logging tutorial](DOCS_BASE_URL_PLACEHOLDER/tutorials/logging-vector-aggregator) + /// to learn how to configure log aggregation with Vector. + #[serde(skip_serializing_if = "Option::is_none")] + pub vector_aggregator_config_map_name: Option, + + /// Extra volumes similar to `.spec.volumes` on a Pod to mount into every container, this can be useful to for + /// example make client certificates, keytabs or similar things available to processors. These volumes will be + /// mounted into all pods at `/stackable/userdata/{volumename}`. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[schemars(schema_with = "raw_object_list_schema")] + pub extra_volumes: Vec, + + /// This field controls which type of Service the Operator creates for this DruidCluster: + /// + /// * `cluster-internal`: Use a ClusterIP service + /// * `external-unstable`: Use a NodePort service + /// * `external-stable`: Use a LoadBalancer service + /// + /// This is a temporary solution with the goal to keep yaml manifests forward compatible. + /// In the future, this setting will control which + /// [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) + /// will be used to expose the service, and ListenerClass names will stay the same, allowing for a non-breaking change. + #[serde(default)] + pub listener_class: CurrentlySupportedListenerClasses, + } } #[derive( @@ -249,82 +325,6 @@ pub enum Container { Vector, } -#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct DruidClusterConfig { - /// Additional extensions to load in Druid. - /// The operator will automatically load all extensions needed based on the cluster - /// configuration, but for extra functionality which the operator cannot anticipate, it can - /// sometimes be necessary to load additional extensions. - /// Add configuration for additional extensions using [configuration override for Druid](https://docs.stackable.tech/home/stable/druid/usage-guide/configuration-and-environment-overrides). - #[serde(default)] - pub additional_extensions: HashSet, - - /// List of [AuthenticationClasses](DOCS_BASE_URL_PLACEHOLDER/concepts/authentication) - /// to use for authenticating users. TLS, LDAP and OIDC authentication are supported. More information in - /// the [Druid operator security documentation](DOCS_BASE_URL_PLACEHOLDER/druid/usage-guide/security#_authentication). - /// - /// For TLS: Please note that the SecretClass used to authenticate users needs to be the same - /// as the SecretClass used for internal communication. - #[serde(default)] - pub authentication: Vec, - - /// Authorization settings for Druid like OPA - #[serde(skip_serializing_if = "Option::is_none")] - pub authorization: Option, - - /// [Druid deep storage configuration](DOCS_BASE_URL_PLACEHOLDER/druid/usage-guide/deep-storage). - /// Only one backend can be used at a time. Either HDFS or S3 are supported. - pub deep_storage: DeepStorageSpec, - - /// Configuration properties for data ingestion tasks. - #[serde(skip_serializing_if = "Option::is_none")] - pub ingestion: Option, - - /// Druid requires an SQL database to store metadata into. Specify connection information here. - pub metadata_storage_database: DatabaseConnectionSpec, - - /// TLS encryption settings for Druid, more information in the - /// [security documentation](DOCS_BASE_URL_PLACEHOLDER/druid/usage-guide/security). - /// This setting only affects server and internal communication. - /// It does not affect client tls authentication, use `clusterConfig.authentication` instead. - #[serde(default = "default_druid_tls", skip_serializing_if = "Option::is_none")] - pub tls: Option, - - /// Druid requires a ZooKeeper cluster connection to run. - /// Provide the name of the ZooKeeper [discovery ConfigMap](DOCS_BASE_URL_PLACEHOLDER/concepts/service_discovery) - /// here. When using the [Stackable operator for Apache ZooKeeper](DOCS_BASE_URL_PLACEHOLDER/zookeeper/) - /// to deploy a ZooKeeper cluster, this will simply be the name of your ZookeeperCluster resource. - pub zookeeper_config_map_name: String, - - /// Name of the Vector aggregator [discovery ConfigMap](DOCS_BASE_URL_PLACEHOLDER/concepts/service_discovery). - /// It must contain the key `ADDRESS` with the address of the Vector aggregator. - /// Follow the [logging tutorial](DOCS_BASE_URL_PLACEHOLDER/tutorials/logging-vector-aggregator) - /// to learn how to configure log aggregation with Vector. - #[serde(skip_serializing_if = "Option::is_none")] - pub vector_aggregator_config_map_name: Option, - - /// Extra volumes similar to `.spec.volumes` on a Pod to mount into every container, this can be useful to for - /// example make client certificates, keytabs or similar things available to processors. These volumes will be - /// mounted into all pods at `/stackable/userdata/{volumename}`. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - #[schemars(schema_with = "raw_object_list_schema")] - pub extra_volumes: Vec, - - /// This field controls which type of Service the Operator creates for this DruidCluster: - /// - /// * `cluster-internal`: Use a ClusterIP service - /// * `external-unstable`: Use a NodePort service - /// * `external-stable`: Use a LoadBalancer service - /// - /// This is a temporary solution with the goal to keep yaml manifests forward compatible. - /// In the future, this setting will control which - /// [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) - /// will be used to expose the service, and ListenerClass names will stay the same, allowing for a non-breaking change. - #[serde(default)] - pub listener_class: CurrentlySupportedListenerClasses, -} - // TODO: Temporary solution until listener-operator is finished #[derive(Clone, Debug, Default, Display, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "PascalCase")] diff --git a/rust/operator-binary/src/crd/tls.rs b/rust/operator-binary/src/crd/tls.rs index 34abdd20..b4fc25b3 100644 --- a/rust/operator-binary/src/crd/tls.rs +++ b/rust/operator-binary/src/crd/tls.rs @@ -33,7 +33,7 @@ pub fn tls_default() -> Option { mod tests { use indoc::formatdoc; - use crate::crd::{tests::deserialize_yaml_str, tls::DruidTls, DruidClusterConfig}; + use crate::crd::{tests::deserialize_yaml_str, tls::DruidTls, v1alpha1::DruidClusterConfig}; const BASE_DRUID_CONFIGURATION: &str = r#" deepStorage: From 5efff59c090249319e27b1b537c75064a56176a8 Mon Sep 17 00:00:00 2001 From: Techassi Date: Tue, 4 Feb 2025 13:27:49 +0100 Subject: [PATCH 09/10] chore: Move DruidCluster impl blocks --- rust/operator-binary/src/crd/mod.rs | 1120 +++++++++++++-------------- 1 file changed, 560 insertions(+), 560 deletions(-) diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 9578e0f7..474966a5 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -304,406 +304,91 @@ pub mod versioned { } } -#[derive( - Clone, - Debug, - Deserialize, - Display, - Eq, - EnumIter, - JsonSchema, - Ord, - PartialEq, - PartialOrd, - Serialize, -)] -#[serde(rename_all = "kebab-case")] -#[strum(serialize_all = "kebab-case")] -pub enum Container { - Druid, - Prepare, - Vector, -} - -// TODO: Temporary solution until listener-operator is finished -#[derive(Clone, Debug, Default, Display, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] -#[serde(rename_all = "PascalCase")] -pub enum CurrentlySupportedListenerClasses { - #[default] - #[serde(rename = "cluster-internal")] - ClusterInternal, - #[serde(rename = "external-unstable")] - ExternalUnstable, - #[serde(rename = "external-stable")] - ExternalStable, -} - -impl CurrentlySupportedListenerClasses { - pub fn k8s_service_type(&self) -> String { - match self { - CurrentlySupportedListenerClasses::ClusterInternal => "ClusterIP".to_string(), - CurrentlySupportedListenerClasses::ExternalUnstable => "NodePort".to_string(), - CurrentlySupportedListenerClasses::ExternalStable => "LoadBalancer".to_string(), +// Required to retrieve the conditions from the cluster status +impl HasStatusCondition for v1alpha1::DruidCluster { + fn conditions(&self) -> Vec { + match &self.status { + Some(status) => status.conditions.clone(), + None => vec![], } } } -/// Common configuration for all role groups -pub struct CommonRoleGroupConfig { - pub resources: RoleResource, - pub logging: Logging, - pub replicas: Option, - pub affinity: StackableAffinity, - pub graceful_shutdown_timeout: Option, - pub requested_secret_lifetime: Duration, -} +impl v1alpha1::DruidCluster { + pub fn common_compute_files( + &self, + file: &str, + ) -> Result>, ConfigError> { + let mut result = BTreeMap::new(); + match file { + JVM_CONFIG => {} + RUNTIME_PROPS => { + let mds = &self.spec.cluster_config.metadata_storage_database; + result.insert( + METADATA_STORAGE_TYPE.to_string(), + Some(mds.db_type.to_string()), + ); + result.insert( + METADATA_STORAGE_URI.to_string(), + Some(mds.conn_string.to_string()), + ); + result.insert( + METADATA_STORAGE_HOST.to_string(), + Some(mds.host.to_string()), + ); + result.insert( + METADATA_STORAGE_PORT.to_string(), + Some(mds.port.to_string()), + ); + if mds.credentials_secret.is_some() { + result.insert( + METADATA_STORAGE_USER.to_string(), + Some(format!("${{env:{DB_USERNAME_ENV}}}")), + ); + result.insert( + METADATA_STORAGE_PASSWORD.to_string(), + Some(format!("${{env:{DB_PASSWORD_ENV}}}")), + ); + } -/// Container for the merged and validated role group configurations -/// -/// This structure contains for every role a map from the role group names to their configurations. -/// The role group configurations are merged with the role and default configurations. The product -/// configuration is not applied. -pub struct MergedConfig { - /// Merged configuration of the broker role - pub brokers: HashMap>, - /// Merged configuration of the coordinator role - pub coordinators: - HashMap>, - /// Merged configuration of the historical role - pub historicals: - HashMap>, - /// Merged configuration of the middle manager role - pub middle_managers: - HashMap>, - /// Merged configuration of the router role - pub routers: HashMap>, -} + // OPA + if let Some(DruidAuthorization { opa: _ }) = &self.spec.cluster_config.authorization + { + result.insert( + AUTH_AUTHORIZERS.to_string(), + Some(AUTH_AUTHORIZERS_VALUE.to_string()), + ); + result.insert( + AUTH_AUTHORIZER_OPA_TYPE.to_string(), + Some(AUTH_AUTHORIZER_OPA_TYPE_VALUE.to_string()), + ); + // The opaUri still needs to be set, but that requires a discovery config map and is handled in the druid_controller.rs + } + // deep storage + result.insert( + DS_TYPE.to_string(), + Some(self.spec.cluster_config.deep_storage.to_string()), + ); + match self.spec.cluster_config.deep_storage.clone() { + DeepStorageSpec::Hdfs(hdfs) => { + result.insert(DS_DIRECTORY.to_string(), Some(hdfs.directory)); + } + DeepStorageSpec::S3(s3_spec) => { + if let Some(key) = &s3_spec.base_key { + result.insert(DS_BASE_KEY.to_string(), Some(key.to_string())); + } + // bucket information (name, connection) needs to be resolved first, + // that is done directly in the controller + } + } -impl MergedConfig { - /// Returns the common configuration for the given role and rolegroup name - pub fn common_config( - &self, - role: DruidRole, - rolegroup_name: &str, - ) -> Result { - match role { - DruidRole::Broker => { - let rolegroup = self - .brokers - .get(rolegroup_name) - .context(CannotRetrieveRoleGroupSnafu { rolegroup_name })?; - Ok(CommonRoleGroupConfig { - resources: RoleResource::Druid(rolegroup.config.config.resources.to_owned()), - logging: rolegroup.config.config.logging.to_owned(), - replicas: rolegroup.replicas, - affinity: rolegroup.config.config.affinity.clone(), - graceful_shutdown_timeout: rolegroup.config.config.graceful_shutdown_timeout, - requested_secret_lifetime: rolegroup - .config - .config - .requested_secret_lifetime - .context(MissingSecretLifetimeSnafu)?, - }) - } - DruidRole::Coordinator => { - let rolegroup = self - .coordinators - .get(rolegroup_name) - .context(CannotRetrieveRoleGroupSnafu { rolegroup_name })?; - Ok(CommonRoleGroupConfig { - resources: RoleResource::Druid(rolegroup.config.config.resources.to_owned()), - logging: rolegroup.config.config.logging.to_owned(), - replicas: rolegroup.replicas, - affinity: rolegroup.config.config.affinity.clone(), - graceful_shutdown_timeout: rolegroup.config.config.graceful_shutdown_timeout, - requested_secret_lifetime: rolegroup - .config - .config - .requested_secret_lifetime - .context(MissingSecretLifetimeSnafu)?, - }) - } - DruidRole::Historical => { - let rolegroup = self - .historicals - .get(rolegroup_name) - .context(CannotRetrieveRoleGroupSnafu { rolegroup_name })?; - Ok(CommonRoleGroupConfig { - resources: RoleResource::Historical( - rolegroup.config.config.resources.to_owned(), - ), - logging: rolegroup.config.config.logging.to_owned(), - replicas: rolegroup.replicas, - affinity: rolegroup.config.config.affinity.clone(), - graceful_shutdown_timeout: rolegroup.config.config.graceful_shutdown_timeout, - requested_secret_lifetime: rolegroup - .config - .config - .requested_secret_lifetime - .context(MissingSecretLifetimeSnafu)?, - }) - } - DruidRole::MiddleManager => { - let rolegroup = self - .middle_managers - .get(rolegroup_name) - .context(CannotRetrieveRoleGroupSnafu { rolegroup_name })?; - Ok(CommonRoleGroupConfig { - resources: RoleResource::Druid(rolegroup.config.config.resources.to_owned()), - logging: rolegroup.config.config.logging.to_owned(), - replicas: rolegroup.replicas, - affinity: rolegroup.config.config.affinity.clone(), - graceful_shutdown_timeout: rolegroup.config.config.graceful_shutdown_timeout, - requested_secret_lifetime: rolegroup - .config - .config - .requested_secret_lifetime - .context(MissingSecretLifetimeSnafu)?, - }) - } - DruidRole::Router => { - let rolegroup = self - .routers - .get(rolegroup_name) - .context(CannotRetrieveRoleGroupSnafu { rolegroup_name })?; - Ok(CommonRoleGroupConfig { - resources: RoleResource::Druid(rolegroup.config.config.resources.to_owned()), - logging: rolegroup.config.config.logging.to_owned(), - replicas: rolegroup.replicas, - affinity: rolegroup.config.config.affinity.clone(), - graceful_shutdown_timeout: rolegroup.config.config.graceful_shutdown_timeout, - requested_secret_lifetime: rolegroup - .config - .config - .requested_secret_lifetime - .context(MissingSecretLifetimeSnafu)?, - }) + // metrics + result.insert(PROMETHEUS_PORT.to_string(), Some(METRICS_PORT.to_string())); } + _ => {} } - } -} - -#[derive( - Clone, - Debug, - Deserialize, - Display, - EnumIter, - Eq, - Hash, - JsonSchema, - PartialEq, - Serialize, - EnumString, -)] -pub enum DruidRole { - #[strum(serialize = "coordinator")] - Coordinator, - #[strum(serialize = "broker")] - Broker, - #[strum(serialize = "historical")] - Historical, - #[strum(serialize = "middlemanager")] - MiddleManager, - #[strum(serialize = "router")] - Router, -} - -impl DruidRole { - /// Returns the name of the internal druid process name associated with the role. - /// These strings are used by druid internally to identify processes. - fn get_process_name(&self) -> &str { - match &self { - DruidRole::Coordinator => "coordinator", - DruidRole::Broker => "broker", - DruidRole::Historical => "historical", - DruidRole::MiddleManager => "middleManager", - DruidRole::Router => "router", - } - } - - /// Returns the http port for every role - pub fn get_http_port(&self) -> u16 { - match &self { - DruidRole::Coordinator => 8081, - DruidRole::Broker => 8082, - DruidRole::Historical => 8083, - DruidRole::MiddleManager => 8091, - DruidRole::Router => 8888, - } - } - - /// Returns the https port for every role - pub fn get_https_port(&self) -> u16 { - match &self { - DruidRole::Coordinator => 8281, - DruidRole::Broker => 8282, - DruidRole::Historical => 8283, - DruidRole::MiddleManager => 8291, - DruidRole::Router => 9088, - } - } - - /// Return the default graceful shutdown timeout - pub fn default_graceful_shutdown_timeout(&self) -> Duration { - match &self { - DruidRole::Coordinator => DEFAULT_COORDINATOR_GRACEFUL_SHUTDOWN_TIMEOUT, - DruidRole::Broker => DEFAULT_BROKER_GRACEFUL_SHUTDOWN_TIMEOUT, - DruidRole::Historical => DEFAULT_HISTORICAL_GRACEFUL_SHUTDOWN_TIMEOUT, - DruidRole::MiddleManager => DEFAULT_MIDDLEMANAGER_GRACEFUL_SHUTDOWN_TIMEOUT, - DruidRole::Router => DEFAULT_ROUTER_GRACEFUL_SHUTDOWN_TIMEOUT, - } - } - - pub fn main_container_prepare_commands( - &self, - s3: Option<&ResolvedS3Connection>, - ) -> Vec { - let mut commands = vec![]; - - if let Some(s3) = s3 { - if let Some(ca_cert_file) = s3.tls.tls_ca_cert_mount_path() { - // The alias can not clash, as we only support a single S3Connection - commands.push(format!("keytool -importcert -file {ca_cert_file} -alias stackable-s3-ca-cert -keystore {STACKABLE_TRUST_STORE} -storepass {STACKABLE_TRUST_STORE_PASSWORD} -noprompt")); - } - } - - // copy druid config to rw config - commands.push(format!( - "cp -RL {conf}/* {rw_conf}", - conf = DRUID_CONFIG_DIRECTORY, - rw_conf = RW_CONFIG_DIRECTORY - )); - - // copy log config to rw config - commands.push(format!( - "cp -RL {conf}/* {rw_conf}", - conf = LOG_CONFIG_DIRECTORY, - rw_conf = RW_CONFIG_DIRECTORY - )); - - // copy hdfs config to RW_CONFIG_DIRECTORY folder (if available) - commands.push(format!( - "cp -RL {hdfs_conf}/* {rw_conf} 2>/dev/null || :", // NOTE: the OR part is here because the command is not applicable sometimes, and would stop everything else from executing - hdfs_conf = HDFS_CONFIG_DIRECTORY, - rw_conf = RW_CONFIG_DIRECTORY, - )); - - commands.extend([ - format!("config-utils template {RW_CONFIG_DIRECTORY}/runtime.properties",), - format!("if test -f {RW_CONFIG_DIRECTORY}/core-site.xml; then config-utils template {RW_CONFIG_DIRECTORY}/core-site.xml; fi",), - format!("if test -f {RW_CONFIG_DIRECTORY}/hdfs-site.xml; then config-utils template {RW_CONFIG_DIRECTORY}/hdfs-site.xml; fi",), - ]); - - commands - } - - pub fn main_container_start_command(&self) -> String { - // We need to store the druid process PID for the graceful shutdown lifecycle pre stop hook. - formatdoc! {" - {COMMON_BASH_TRAP_FUNCTIONS} - {remove_vector_shutdown_file_command} - prepare_signal_handlers - containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & - /stackable/druid/bin/run-druid {process_name} {RW_CONFIG_DIRECTORY} & - echo \"$!\" >> /tmp/DRUID_PID - wait_for_termination $(cat /tmp/DRUID_PID) - {create_vector_shutdown_file_command} - ", - process_name = self.get_process_name(), - remove_vector_shutdown_file_command = - remove_vector_shutdown_file_command(STACKABLE_LOG_DIR), - create_vector_shutdown_file_command = - create_vector_shutdown_file_command(STACKABLE_LOG_DIR), - } - } -} - -// Required to retrieve the conditions from the cluster status -impl HasStatusCondition for v1alpha1::DruidCluster { - fn conditions(&self) -> Vec { - match &self.status { - Some(status) => status.conditions.clone(), - None => vec![], - } - } -} - -impl v1alpha1::DruidCluster { - pub fn common_compute_files( - &self, - file: &str, - ) -> Result>, ConfigError> { - let mut result = BTreeMap::new(); - match file { - JVM_CONFIG => {} - RUNTIME_PROPS => { - let mds = &self.spec.cluster_config.metadata_storage_database; - result.insert( - METADATA_STORAGE_TYPE.to_string(), - Some(mds.db_type.to_string()), - ); - result.insert( - METADATA_STORAGE_URI.to_string(), - Some(mds.conn_string.to_string()), - ); - result.insert( - METADATA_STORAGE_HOST.to_string(), - Some(mds.host.to_string()), - ); - result.insert( - METADATA_STORAGE_PORT.to_string(), - Some(mds.port.to_string()), - ); - if mds.credentials_secret.is_some() { - result.insert( - METADATA_STORAGE_USER.to_string(), - Some(format!("${{env:{DB_USERNAME_ENV}}}")), - ); - result.insert( - METADATA_STORAGE_PASSWORD.to_string(), - Some(format!("${{env:{DB_PASSWORD_ENV}}}")), - ); - } - - // OPA - if let Some(DruidAuthorization { opa: _ }) = &self.spec.cluster_config.authorization - { - result.insert( - AUTH_AUTHORIZERS.to_string(), - Some(AUTH_AUTHORIZERS_VALUE.to_string()), - ); - result.insert( - AUTH_AUTHORIZER_OPA_TYPE.to_string(), - Some(AUTH_AUTHORIZER_OPA_TYPE_VALUE.to_string()), - ); - // The opaUri still needs to be set, but that requires a discovery config map and is handled in the druid_controller.rs - } - // deep storage - result.insert( - DS_TYPE.to_string(), - Some(self.spec.cluster_config.deep_storage.to_string()), - ); - match self.spec.cluster_config.deep_storage.clone() { - DeepStorageSpec::Hdfs(hdfs) => { - result.insert(DS_DIRECTORY.to_string(), Some(hdfs.directory)); - } - DeepStorageSpec::S3(s3_spec) => { - if let Some(key) = &s3_spec.base_key { - result.insert(DS_BASE_KEY.to_string(), Some(key.to_string())); - } - // bucket information (name, connection) needs to be resolved first, - // that is done directly in the controller - } - } - - // metrics - result.insert(PROMETHEUS_PORT.to_string(), Some(METRICS_PORT.to_string())); - } - _ => {} - } - - Ok(result) + + Ok(result) } pub fn build_role_properties( @@ -840,189 +525,504 @@ impl v1alpha1::DruidCluster { } } - /// Returns true if the cluster uses an s3 connection. - /// This is a quicker convenience function over the [v1alpha1::DruidCluster::get_s3_connection] function. - pub fn uses_s3(&self) -> bool { - let s3_ingestion = self - .spec - .cluster_config - .ingestion - .as_ref() - .and_then(|spec| spec.s3connection.as_ref()) - .is_some(); - let s3_storage = self.spec.cluster_config.deep_storage.is_s3(); - s3_ingestion || s3_storage + /// Returns true if the cluster uses an s3 connection. + /// This is a quicker convenience function over the [v1alpha1::DruidCluster::get_s3_connection] function. + pub fn uses_s3(&self) -> bool { + let s3_ingestion = self + .spec + .cluster_config + .ingestion + .as_ref() + .and_then(|spec| spec.s3connection.as_ref()) + .is_some(); + let s3_storage = self.spec.cluster_config.deep_storage.is_s3(); + s3_ingestion || s3_storage + } + + /// Returns the merged and validated configuration for all roles + pub fn merged_config(&self) -> Result { + let deep_storage = &self.spec.cluster_config.deep_storage; + + Ok(MergedConfig { + brokers: v1alpha1::DruidCluster::merged_role( + &self.spec.brokers, + &BrokerConfig::default_config(&self.name_any(), &DruidRole::Broker, deep_storage), + )?, + coordinators: v1alpha1::DruidCluster::merged_role( + &self.spec.coordinators, + &CoordinatorConfig::default_config( + &self.name_any(), + &DruidRole::Coordinator, + deep_storage, + ), + )?, + historicals: v1alpha1::DruidCluster::merged_role( + &self.spec.historicals, + &HistoricalConfig::default_config( + &self.name_any(), + &DruidRole::Historical, + deep_storage, + ), + )?, + middle_managers: v1alpha1::DruidCluster::merged_role( + &self.spec.middle_managers, + &MiddleManagerConfig::default_config( + &self.name_any(), + &DruidRole::MiddleManager, + deep_storage, + ), + )?, + routers: v1alpha1::DruidCluster::merged_role( + &self.spec.routers, + &RouterConfig::default_config(&self.name_any(), &DruidRole::Router, deep_storage), + )?, + }) + } + + /// Merges and validates the role groups of the given role with the given default configuration + fn merged_role( + role: &Role, + default_config: &T::Fragment, + ) -> Result>, Error> + where + T: FromFragment, + T::Fragment: Clone + Merge, + { + let mut merged_role_config = HashMap::new(); + + for (rolegroup_name, rolegroup) in &role.role_groups { + let merged_rolegroup_config = v1alpha1::DruidCluster::merged_rolegroup( + rolegroup, + &role.config.config, + default_config, + )?; + merged_role_config.insert(rolegroup_name.to_owned(), merged_rolegroup_config); + } + + Ok(merged_role_config) + } + + /// Merges and validates the given role group with the given role and default configurations + fn merged_rolegroup( + rolegroup: &RoleGroup, + role_config: &T::Fragment, + default_config: &T::Fragment, + ) -> Result, Error> + where + T: FromFragment, + T::Fragment: Clone + Merge, + { + let merged_config = v1alpha1::DruidCluster::merged_rolegroup_config( + &rolegroup.config.config, + role_config, + default_config, + )?; + Ok(RoleGroup { + config: CommonConfiguration { + config: merged_config, + config_overrides: rolegroup.config.config_overrides.to_owned(), + env_overrides: rolegroup.config.env_overrides.to_owned(), + cli_overrides: rolegroup.config.cli_overrides.to_owned(), + pod_overrides: rolegroup.config.pod_overrides.to_owned(), + product_specific_common_config: rolegroup + .config + .product_specific_common_config + .to_owned(), + }, + replicas: rolegroup.replicas, + }) + } + + pub fn role_config(&self, role: &DruidRole) -> &GenericRoleConfig { + match role { + DruidRole::Broker => &self.spec.brokers.role_config, + DruidRole::Coordinator => &self.spec.coordinators.role_config, + DruidRole::Historical => &self.spec.historicals.role_config, + DruidRole::MiddleManager => &self.spec.middle_managers.role_config, + DruidRole::Router => &self.spec.routers.role_config, + } + } + + /// Merges and validates the given role group, role, and default configurations + pub fn merged_rolegroup_config( + rolegroup_config: &T::Fragment, + role_config: &T::Fragment, + default_config: &T::Fragment, + ) -> Result + where + T: FromFragment, + T::Fragment: Clone + Merge, + { + let mut role_config = role_config.to_owned(); + let mut rolegroup_config = rolegroup_config.to_owned(); + + role_config.merge(default_config); + rolegroup_config.merge(&role_config); + + fragment::validate(rolegroup_config).context(FragmentValidationFailureSnafu) + } + + pub fn pod_overrides_for_role(&self, role: &DruidRole) -> &PodTemplateSpec { + match role { + DruidRole::Broker => &self.spec.brokers.config.pod_overrides, + DruidRole::Coordinator => &self.spec.coordinators.config.pod_overrides, + DruidRole::Historical => &self.spec.historicals.config.pod_overrides, + DruidRole::MiddleManager => &self.spec.middle_managers.config.pod_overrides, + DruidRole::Router => &self.spec.routers.config.pod_overrides, + } + } + + pub fn pod_overrides_for_role_group( + &self, + role: &DruidRole, + role_group: &str, + ) -> Option<&PodTemplateSpec> { + match role { + DruidRole::Broker => self + .spec + .brokers + .role_groups + .get(role_group) + .map(|rg| &rg.config.pod_overrides), + DruidRole::Coordinator => self + .spec + .coordinators + .role_groups + .get(role_group) + .map(|rg| &rg.config.pod_overrides), + DruidRole::Historical => self + .spec + .historicals + .role_groups + .get(role_group) + .map(|rg| &rg.config.pod_overrides), + DruidRole::MiddleManager => self + .spec + .middle_managers + .role_groups + .get(role_group) + .map(|rg| &rg.config.pod_overrides), + DruidRole::Router => self + .spec + .routers + .role_groups + .get(role_group) + .map(|rg| &rg.config.pod_overrides), + } + } +} + +#[derive( + Clone, + Debug, + Deserialize, + Display, + Eq, + EnumIter, + JsonSchema, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[serde(rename_all = "kebab-case")] +#[strum(serialize_all = "kebab-case")] +pub enum Container { + Druid, + Prepare, + Vector, +} + +// TODO: Temporary solution until listener-operator is finished +#[derive(Clone, Debug, Default, Display, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "PascalCase")] +pub enum CurrentlySupportedListenerClasses { + #[default] + #[serde(rename = "cluster-internal")] + ClusterInternal, + #[serde(rename = "external-unstable")] + ExternalUnstable, + #[serde(rename = "external-stable")] + ExternalStable, +} + +impl CurrentlySupportedListenerClasses { + pub fn k8s_service_type(&self) -> String { + match self { + CurrentlySupportedListenerClasses::ClusterInternal => "ClusterIP".to_string(), + CurrentlySupportedListenerClasses::ExternalUnstable => "NodePort".to_string(), + CurrentlySupportedListenerClasses::ExternalStable => "LoadBalancer".to_string(), + } + } +} + +/// Common configuration for all role groups +pub struct CommonRoleGroupConfig { + pub resources: RoleResource, + pub logging: Logging, + pub replicas: Option, + pub affinity: StackableAffinity, + pub graceful_shutdown_timeout: Option, + pub requested_secret_lifetime: Duration, +} + +/// Container for the merged and validated role group configurations +/// +/// This structure contains for every role a map from the role group names to their configurations. +/// The role group configurations are merged with the role and default configurations. The product +/// configuration is not applied. +pub struct MergedConfig { + /// Merged configuration of the broker role + pub brokers: HashMap>, + /// Merged configuration of the coordinator role + pub coordinators: + HashMap>, + /// Merged configuration of the historical role + pub historicals: + HashMap>, + /// Merged configuration of the middle manager role + pub middle_managers: + HashMap>, + /// Merged configuration of the router role + pub routers: HashMap>, +} + +impl MergedConfig { + /// Returns the common configuration for the given role and rolegroup name + pub fn common_config( + &self, + role: DruidRole, + rolegroup_name: &str, + ) -> Result { + match role { + DruidRole::Broker => { + let rolegroup = self + .brokers + .get(rolegroup_name) + .context(CannotRetrieveRoleGroupSnafu { rolegroup_name })?; + Ok(CommonRoleGroupConfig { + resources: RoleResource::Druid(rolegroup.config.config.resources.to_owned()), + logging: rolegroup.config.config.logging.to_owned(), + replicas: rolegroup.replicas, + affinity: rolegroup.config.config.affinity.clone(), + graceful_shutdown_timeout: rolegroup.config.config.graceful_shutdown_timeout, + requested_secret_lifetime: rolegroup + .config + .config + .requested_secret_lifetime + .context(MissingSecretLifetimeSnafu)?, + }) + } + DruidRole::Coordinator => { + let rolegroup = self + .coordinators + .get(rolegroup_name) + .context(CannotRetrieveRoleGroupSnafu { rolegroup_name })?; + Ok(CommonRoleGroupConfig { + resources: RoleResource::Druid(rolegroup.config.config.resources.to_owned()), + logging: rolegroup.config.config.logging.to_owned(), + replicas: rolegroup.replicas, + affinity: rolegroup.config.config.affinity.clone(), + graceful_shutdown_timeout: rolegroup.config.config.graceful_shutdown_timeout, + requested_secret_lifetime: rolegroup + .config + .config + .requested_secret_lifetime + .context(MissingSecretLifetimeSnafu)?, + }) + } + DruidRole::Historical => { + let rolegroup = self + .historicals + .get(rolegroup_name) + .context(CannotRetrieveRoleGroupSnafu { rolegroup_name })?; + Ok(CommonRoleGroupConfig { + resources: RoleResource::Historical( + rolegroup.config.config.resources.to_owned(), + ), + logging: rolegroup.config.config.logging.to_owned(), + replicas: rolegroup.replicas, + affinity: rolegroup.config.config.affinity.clone(), + graceful_shutdown_timeout: rolegroup.config.config.graceful_shutdown_timeout, + requested_secret_lifetime: rolegroup + .config + .config + .requested_secret_lifetime + .context(MissingSecretLifetimeSnafu)?, + }) + } + DruidRole::MiddleManager => { + let rolegroup = self + .middle_managers + .get(rolegroup_name) + .context(CannotRetrieveRoleGroupSnafu { rolegroup_name })?; + Ok(CommonRoleGroupConfig { + resources: RoleResource::Druid(rolegroup.config.config.resources.to_owned()), + logging: rolegroup.config.config.logging.to_owned(), + replicas: rolegroup.replicas, + affinity: rolegroup.config.config.affinity.clone(), + graceful_shutdown_timeout: rolegroup.config.config.graceful_shutdown_timeout, + requested_secret_lifetime: rolegroup + .config + .config + .requested_secret_lifetime + .context(MissingSecretLifetimeSnafu)?, + }) + } + DruidRole::Router => { + let rolegroup = self + .routers + .get(rolegroup_name) + .context(CannotRetrieveRoleGroupSnafu { rolegroup_name })?; + Ok(CommonRoleGroupConfig { + resources: RoleResource::Druid(rolegroup.config.config.resources.to_owned()), + logging: rolegroup.config.config.logging.to_owned(), + replicas: rolegroup.replicas, + affinity: rolegroup.config.config.affinity.clone(), + graceful_shutdown_timeout: rolegroup.config.config.graceful_shutdown_timeout, + requested_secret_lifetime: rolegroup + .config + .config + .requested_secret_lifetime + .context(MissingSecretLifetimeSnafu)?, + }) + } + } + } +} + +#[derive( + Clone, + Debug, + Deserialize, + Display, + EnumIter, + Eq, + Hash, + JsonSchema, + PartialEq, + Serialize, + EnumString, +)] +pub enum DruidRole { + #[strum(serialize = "coordinator")] + Coordinator, + #[strum(serialize = "broker")] + Broker, + #[strum(serialize = "historical")] + Historical, + #[strum(serialize = "middlemanager")] + MiddleManager, + #[strum(serialize = "router")] + Router, +} + +impl DruidRole { + /// Returns the name of the internal druid process name associated with the role. + /// These strings are used by druid internally to identify processes. + fn get_process_name(&self) -> &str { + match &self { + DruidRole::Coordinator => "coordinator", + DruidRole::Broker => "broker", + DruidRole::Historical => "historical", + DruidRole::MiddleManager => "middleManager", + DruidRole::Router => "router", + } + } + + /// Returns the http port for every role + pub fn get_http_port(&self) -> u16 { + match &self { + DruidRole::Coordinator => 8081, + DruidRole::Broker => 8082, + DruidRole::Historical => 8083, + DruidRole::MiddleManager => 8091, + DruidRole::Router => 8888, + } } - /// Returns the merged and validated configuration for all roles - pub fn merged_config(&self) -> Result { - let deep_storage = &self.spec.cluster_config.deep_storage; - - Ok(MergedConfig { - brokers: v1alpha1::DruidCluster::merged_role( - &self.spec.brokers, - &BrokerConfig::default_config(&self.name_any(), &DruidRole::Broker, deep_storage), - )?, - coordinators: v1alpha1::DruidCluster::merged_role( - &self.spec.coordinators, - &CoordinatorConfig::default_config( - &self.name_any(), - &DruidRole::Coordinator, - deep_storage, - ), - )?, - historicals: v1alpha1::DruidCluster::merged_role( - &self.spec.historicals, - &HistoricalConfig::default_config( - &self.name_any(), - &DruidRole::Historical, - deep_storage, - ), - )?, - middle_managers: v1alpha1::DruidCluster::merged_role( - &self.spec.middle_managers, - &MiddleManagerConfig::default_config( - &self.name_any(), - &DruidRole::MiddleManager, - deep_storage, - ), - )?, - routers: v1alpha1::DruidCluster::merged_role( - &self.spec.routers, - &RouterConfig::default_config(&self.name_any(), &DruidRole::Router, deep_storage), - )?, - }) + /// Returns the https port for every role + pub fn get_https_port(&self) -> u16 { + match &self { + DruidRole::Coordinator => 8281, + DruidRole::Broker => 8282, + DruidRole::Historical => 8283, + DruidRole::MiddleManager => 8291, + DruidRole::Router => 9088, + } } - /// Merges and validates the role groups of the given role with the given default configuration - fn merged_role( - role: &Role, - default_config: &T::Fragment, - ) -> Result>, Error> - where - T: FromFragment, - T::Fragment: Clone + Merge, - { - let mut merged_role_config = HashMap::new(); - - for (rolegroup_name, rolegroup) in &role.role_groups { - let merged_rolegroup_config = v1alpha1::DruidCluster::merged_rolegroup( - rolegroup, - &role.config.config, - default_config, - )?; - merged_role_config.insert(rolegroup_name.to_owned(), merged_rolegroup_config); + /// Return the default graceful shutdown timeout + pub fn default_graceful_shutdown_timeout(&self) -> Duration { + match &self { + DruidRole::Coordinator => DEFAULT_COORDINATOR_GRACEFUL_SHUTDOWN_TIMEOUT, + DruidRole::Broker => DEFAULT_BROKER_GRACEFUL_SHUTDOWN_TIMEOUT, + DruidRole::Historical => DEFAULT_HISTORICAL_GRACEFUL_SHUTDOWN_TIMEOUT, + DruidRole::MiddleManager => DEFAULT_MIDDLEMANAGER_GRACEFUL_SHUTDOWN_TIMEOUT, + DruidRole::Router => DEFAULT_ROUTER_GRACEFUL_SHUTDOWN_TIMEOUT, } - - Ok(merged_role_config) } - /// Merges and validates the given role group with the given role and default configurations - fn merged_rolegroup( - rolegroup: &RoleGroup, - role_config: &T::Fragment, - default_config: &T::Fragment, - ) -> Result, Error> - where - T: FromFragment, - T::Fragment: Clone + Merge, - { - let merged_config = v1alpha1::DruidCluster::merged_rolegroup_config( - &rolegroup.config.config, - role_config, - default_config, - )?; - Ok(RoleGroup { - config: CommonConfiguration { - config: merged_config, - config_overrides: rolegroup.config.config_overrides.to_owned(), - env_overrides: rolegroup.config.env_overrides.to_owned(), - cli_overrides: rolegroup.config.cli_overrides.to_owned(), - pod_overrides: rolegroup.config.pod_overrides.to_owned(), - product_specific_common_config: rolegroup - .config - .product_specific_common_config - .to_owned(), - }, - replicas: rolegroup.replicas, - }) - } + pub fn main_container_prepare_commands( + &self, + s3: Option<&ResolvedS3Connection>, + ) -> Vec { + let mut commands = vec![]; - pub fn role_config(&self, role: &DruidRole) -> &GenericRoleConfig { - match role { - DruidRole::Broker => &self.spec.brokers.role_config, - DruidRole::Coordinator => &self.spec.coordinators.role_config, - DruidRole::Historical => &self.spec.historicals.role_config, - DruidRole::MiddleManager => &self.spec.middle_managers.role_config, - DruidRole::Router => &self.spec.routers.role_config, + if let Some(s3) = s3 { + if let Some(ca_cert_file) = s3.tls.tls_ca_cert_mount_path() { + // The alias can not clash, as we only support a single S3Connection + commands.push(format!("keytool -importcert -file {ca_cert_file} -alias stackable-s3-ca-cert -keystore {STACKABLE_TRUST_STORE} -storepass {STACKABLE_TRUST_STORE_PASSWORD} -noprompt")); + } } - } - /// Merges and validates the given role group, role, and default configurations - pub fn merged_rolegroup_config( - rolegroup_config: &T::Fragment, - role_config: &T::Fragment, - default_config: &T::Fragment, - ) -> Result - where - T: FromFragment, - T::Fragment: Clone + Merge, - { - let mut role_config = role_config.to_owned(); - let mut rolegroup_config = rolegroup_config.to_owned(); + // copy druid config to rw config + commands.push(format!( + "cp -RL {conf}/* {rw_conf}", + conf = DRUID_CONFIG_DIRECTORY, + rw_conf = RW_CONFIG_DIRECTORY + )); - role_config.merge(default_config); - rolegroup_config.merge(&role_config); + // copy log config to rw config + commands.push(format!( + "cp -RL {conf}/* {rw_conf}", + conf = LOG_CONFIG_DIRECTORY, + rw_conf = RW_CONFIG_DIRECTORY + )); - fragment::validate(rolegroup_config).context(FragmentValidationFailureSnafu) - } + // copy hdfs config to RW_CONFIG_DIRECTORY folder (if available) + commands.push(format!( + "cp -RL {hdfs_conf}/* {rw_conf} 2>/dev/null || :", // NOTE: the OR part is here because the command is not applicable sometimes, and would stop everything else from executing + hdfs_conf = HDFS_CONFIG_DIRECTORY, + rw_conf = RW_CONFIG_DIRECTORY, + )); - pub fn pod_overrides_for_role(&self, role: &DruidRole) -> &PodTemplateSpec { - match role { - DruidRole::Broker => &self.spec.brokers.config.pod_overrides, - DruidRole::Coordinator => &self.spec.coordinators.config.pod_overrides, - DruidRole::Historical => &self.spec.historicals.config.pod_overrides, - DruidRole::MiddleManager => &self.spec.middle_managers.config.pod_overrides, - DruidRole::Router => &self.spec.routers.config.pod_overrides, - } + commands.extend([ + format!("config-utils template {RW_CONFIG_DIRECTORY}/runtime.properties",), + format!("if test -f {RW_CONFIG_DIRECTORY}/core-site.xml; then config-utils template {RW_CONFIG_DIRECTORY}/core-site.xml; fi",), + format!("if test -f {RW_CONFIG_DIRECTORY}/hdfs-site.xml; then config-utils template {RW_CONFIG_DIRECTORY}/hdfs-site.xml; fi",), + ]); + + commands } - pub fn pod_overrides_for_role_group( - &self, - role: &DruidRole, - role_group: &str, - ) -> Option<&PodTemplateSpec> { - match role { - DruidRole::Broker => self - .spec - .brokers - .role_groups - .get(role_group) - .map(|rg| &rg.config.pod_overrides), - DruidRole::Coordinator => self - .spec - .coordinators - .role_groups - .get(role_group) - .map(|rg| &rg.config.pod_overrides), - DruidRole::Historical => self - .spec - .historicals - .role_groups - .get(role_group) - .map(|rg| &rg.config.pod_overrides), - DruidRole::MiddleManager => self - .spec - .middle_managers - .role_groups - .get(role_group) - .map(|rg| &rg.config.pod_overrides), - DruidRole::Router => self - .spec - .routers - .role_groups - .get(role_group) - .map(|rg| &rg.config.pod_overrides), + pub fn main_container_start_command(&self) -> String { + // We need to store the druid process PID for the graceful shutdown lifecycle pre stop hook. + formatdoc! {" + {COMMON_BASH_TRAP_FUNCTIONS} + {remove_vector_shutdown_file_command} + prepare_signal_handlers + containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & + /stackable/druid/bin/run-druid {process_name} {RW_CONFIG_DIRECTORY} & + echo \"$!\" >> /tmp/DRUID_PID + wait_for_termination $(cat /tmp/DRUID_PID) + {create_vector_shutdown_file_command} + ", + process_name = self.get_process_name(), + remove_vector_shutdown_file_command = + remove_vector_shutdown_file_command(STACKABLE_LOG_DIR), + create_vector_shutdown_file_command = + create_vector_shutdown_file_command(STACKABLE_LOG_DIR), } } } From 6dd850d9671eee01a3411bc8f70627480fa6fbc6 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 12 Feb 2025 14:22:17 +0100 Subject: [PATCH 10/10] make regenerate-nix --- Cargo.nix | 1361 +++++++++++++++++++++++++++++++++++++-------- crate-hashes.json | 3 + 2 files changed, 1144 insertions(+), 220 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index a223d51c..fd5e1e32 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -32,21 +32,23 @@ rec { # "public" attributes that we attempt to keep stable with new versions of crate2nix. # + rootCrate = rec { + packageId = "stackable-druid-operator"; + # Use this attribute to refer to the derivation building your root crate package. + # You can override the features with rootCrate.build.override { features = [ "default" "feature1" ... ]; }. + build = internal.buildRustCrateWithFeatures { + inherit packageId; + }; + + # Debug support which might change between releases. + # File a bug if you depend on any for non-debug work! + debug = internal.debugCrate { inherit packageId; }; + }; # Refer your crate build derivation by name here. # You can override the features with # workspaceMembers."${crateName}".build.override { features = [ "default" "feature1" ... ]; }. workspaceMembers = { - "stackable-druid-crd" = rec { - packageId = "stackable-druid-crd"; - build = internal.buildRustCrateWithFeatures { - packageId = "stackable-druid-crd"; - }; - - # Debug support which might change between releases. - # File a bug if you depend on any for non-debug work! - debug = internal.debugCrate { inherit packageId; }; - }; "stackable-druid-operator" = rec { packageId = "stackable-druid-operator"; build = internal.buildRustCrateWithFeatures { @@ -655,7 +657,22 @@ rec { }; resolvedDefaultFeatures = [ "std" ]; }; - "bitflags" = rec { + "bitflags 1.3.2" = rec { + crateName = "bitflags"; + version = "1.3.2"; + edition = "2018"; + sha256 = "12ki6w8gn1ldq7yz9y680llwk5gmrhrzszaa17g1sbrw2r2qvwxy"; + authors = [ + "The Rust Project Developers" + ]; + features = { + "compiler_builtins" = [ "dep:compiler_builtins" ]; + "core" = [ "dep:core" ]; + "rustc-dep-of-std" = [ "core" "compiler_builtins" ]; + }; + resolvedDefaultFeatures = [ "default" ]; + }; + "bitflags 2.8.0" = rec { crateName = "bitflags"; version = "2.8.0"; edition = "2021"; @@ -1086,6 +1103,25 @@ rec { }; resolvedDefaultFeatures = [ "default" ]; }; + "convert_case" = rec { + crateName = "convert_case"; + version = "0.6.0"; + edition = "2018"; + sha256 = "1jn1pq6fp3rri88zyw6jlhwwgf6qiyc08d6gjv0qypgkl862n67c"; + authors = [ + "Rutrum " + ]; + dependencies = [ + { + name = "unicode-segmentation"; + packageId = "unicode-segmentation"; + } + ]; + features = { + "rand" = [ "dep:rand" ]; + "random" = [ "rand" ]; + }; + }; "core-foundation 0.10.0" = rec { crateName = "core-foundation"; version = "0.10.0"; @@ -1767,6 +1803,26 @@ rec { }; resolvedDefaultFeatures = [ "default" "perf" "std" "unicode" ]; }; + "fluent-uri" = rec { + crateName = "fluent-uri"; + version = "0.1.4"; + edition = "2021"; + sha256 = "03ah2qajw5l1zbc81kh1n8g7n24mfxbg6vqyv9ixipg1vglh9iqp"; + libName = "fluent_uri"; + authors = [ + "Scallop Ye " + ]; + dependencies = [ + { + name = "bitflags"; + packageId = "bitflags 1.3.2"; + } + ]; + features = { + "default" = [ "std" ]; + }; + resolvedDefaultFeatures = [ "std" ]; + }; "fnv" = rec { crateName = "fnv"; version = "1.0.7"; @@ -2248,7 +2304,7 @@ rec { dependencies = [ { name = "bitflags"; - packageId = "bitflags"; + packageId = "bitflags 2.8.0"; } { name = "libc"; @@ -2289,7 +2345,46 @@ rec { ]; }; - "hashbrown" = rec { + "hashbrown 0.14.5" = rec { + crateName = "hashbrown"; + version = "0.14.5"; + edition = "2021"; + sha256 = "1wa1vy1xs3mp11bn3z9dv0jricgr6a2j0zkf1g19yz3vw4il89z5"; + authors = [ + "Amanieu d'Antras " + ]; + dependencies = [ + { + name = "ahash"; + packageId = "ahash"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "allocator-api2"; + packageId = "allocator-api2"; + optional = true; + usesDefaultFeatures = false; + features = [ "alloc" ]; + } + ]; + features = { + "ahash" = [ "dep:ahash" ]; + "alloc" = [ "dep:alloc" ]; + "allocator-api2" = [ "dep:allocator-api2" ]; + "compiler_builtins" = [ "dep:compiler_builtins" ]; + "core" = [ "dep:core" ]; + "default" = [ "ahash" "inline-more" "allocator-api2" ]; + "equivalent" = [ "dep:equivalent" ]; + "nightly" = [ "allocator-api2?/nightly" "bumpalo/allocator_api" ]; + "rayon" = [ "dep:rayon" ]; + "rkyv" = [ "dep:rkyv" ]; + "rustc-dep-of-std" = [ "nightly" "core" "compiler_builtins" "alloc" "rustc-internal-api" ]; + "serde" = [ "dep:serde" ]; + }; + resolvedDefaultFeatures = [ "ahash" "allocator-api2" "default" "inline-more" ]; + }; + "hashbrown 0.15.2" = rec { crateName = "hashbrown"; version = "0.15.2"; edition = "2021"; @@ -3559,7 +3654,7 @@ rec { } { name = "hashbrown"; - packageId = "hashbrown"; + packageId = "hashbrown 0.15.2"; usesDefaultFeatures = false; } ]; @@ -3633,6 +3728,27 @@ rec { }; resolvedDefaultFeatures = [ "default" ]; }; + "itertools" = rec { + crateName = "itertools"; + version = "0.13.0"; + edition = "2018"; + sha256 = "11hiy3qzl643zcigknclh446qb9zlg4dpdzfkjaa9q9fqpgyfgj1"; + authors = [ + "bluss" + ]; + dependencies = [ + { + name = "either"; + packageId = "either"; + usesDefaultFeatures = false; + } + ]; + features = { + "default" = [ "use_std" ]; + "use_std" = [ "use_alloc" "either/use_std" ]; + }; + resolvedDefaultFeatures = [ "default" "use_alloc" "use_std" ]; + }; "itoa" = rec { crateName = "itoa"; version = "1.0.14"; @@ -3714,7 +3830,48 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; - "json-patch" = rec { + "json-patch 2.0.0" = rec { + crateName = "json-patch"; + version = "2.0.0"; + edition = "2021"; + sha256 = "1z1h6dyy4lx4z74yby2hvgl4jbm8mh5ymjp6fwcdkyi3923bh7sv"; + libName = "json_patch"; + authors = [ + "Ivan Dubrov " + ]; + dependencies = [ + { + name = "jsonptr"; + packageId = "jsonptr 0.4.7"; + } + { + name = "serde"; + packageId = "serde"; + features = [ "derive" ]; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + { + name = "thiserror"; + packageId = "thiserror 1.0.69"; + } + ]; + devDependencies = [ + { + name = "serde_json"; + packageId = "serde_json"; + features = [ "preserve_order" ]; + } + ]; + features = { + "default" = [ "diff" ]; + "utoipa" = [ "dep:utoipa" ]; + }; + resolvedDefaultFeatures = [ "default" "diff" ]; + }; + "json-patch 3.0.1" = rec { crateName = "json-patch"; version = "3.0.1"; edition = "2021"; @@ -3726,7 +3883,7 @@ rec { dependencies = [ { name = "jsonptr"; - packageId = "jsonptr"; + packageId = "jsonptr 0.6.3"; } { name = "serde"; @@ -3755,7 +3912,48 @@ rec { }; resolvedDefaultFeatures = [ "default" "diff" ]; }; - "jsonpath-rust" = rec { + "jsonpath-rust 0.5.1" = rec { + crateName = "jsonpath-rust"; + version = "0.5.1"; + edition = "2018"; + sha256 = "0032bp43w6k1bl8h55m126cdf8xljj8p736f65gp3zvhpn2zxn0r"; + libName = "jsonpath_rust"; + authors = [ + "BorisZhguchev " + ]; + dependencies = [ + { + name = "lazy_static"; + packageId = "lazy_static"; + } + { + name = "once_cell"; + packageId = "once_cell"; + } + { + name = "pest"; + packageId = "pest"; + } + { + name = "pest_derive"; + packageId = "pest_derive"; + } + { + name = "regex"; + packageId = "regex"; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + { + name = "thiserror"; + packageId = "thiserror 1.0.69"; + } + ]; + + }; + "jsonpath-rust 0.7.5" = rec { crateName = "jsonpath-rust"; version = "0.7.5"; edition = "2021"; @@ -3788,7 +3986,44 @@ rec { ]; }; - "jsonptr" = rec { + "jsonptr 0.4.7" = rec { + crateName = "jsonptr"; + version = "0.4.7"; + edition = "2021"; + sha256 = "09s6bqjlkd1m5z9hi9iwjimiri7wx3fd6d88hara0p27968m4vhw"; + authors = [ + "chance dinkins" + ]; + dependencies = [ + { + name = "fluent-uri"; + packageId = "fluent-uri"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "serde"; + packageId = "serde"; + usesDefaultFeatures = false; + features = [ "alloc" ]; + } + { + name = "serde_json"; + packageId = "serde_json"; + usesDefaultFeatures = false; + features = [ "alloc" ]; + } + ]; + features = { + "default" = [ "std" ]; + "fluent-uri" = [ "dep:fluent-uri" ]; + "std" = [ "serde/std" "serde_json/std" "fluent-uri?/std" ]; + "uniresid" = [ "dep:uniresid" ]; + "url" = [ "dep:url" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; + "jsonptr 0.6.3" = rec { crateName = "jsonptr"; version = "0.6.3"; edition = "2021"; @@ -3822,12 +4057,12 @@ rec { }; resolvedDefaultFeatures = [ "assign" "default" "delete" "json" "resolve" "serde" "std" ]; }; - "k8s-openapi" = rec { + "k8s-openapi 0.23.0" = rec { crateName = "k8s-openapi"; - version = "0.24.0"; + version = "0.23.0"; edition = "2021"; - links = "k8s-openapi-0.24.0"; - sha256 = "1m8ahw59g44kp9p4yd4ar0px15m2nyvhc5krbvqvw2ag6a8bjx9c"; + links = "k8s-openapi-0.23.0"; + sha256 = "04qv2iqwm3mgjvyp2m6n3vf6nnpjh5a60kf9ah9k1n184d04g24w"; libName = "k8s_openapi"; authors = [ "Arnav Singh " @@ -3869,62 +4104,149 @@ rec { } ]; features = { - "earliest" = [ "v1_28" ]; - "latest" = [ "v1_32" ]; + "earliest" = [ "v1_26" ]; + "latest" = [ "v1_31" ]; "schemars" = [ "dep:schemars" ]; }; - resolvedDefaultFeatures = [ "schemars" "v1_32" ]; + resolvedDefaultFeatures = [ "schemars" "v1_31" ]; }; - "kube" = rec { - crateName = "kube"; - version = "0.98.0"; + "k8s-openapi 0.24.0" = rec { + crateName = "k8s-openapi"; + version = "0.24.0"; edition = "2021"; - sha256 = "1fiwllwzsvl7921k85c10d1nwjpg09ycqcvvihc4vbggjp23s19j"; + links = "k8s-openapi-0.24.0"; + sha256 = "1m8ahw59g44kp9p4yd4ar0px15m2nyvhc5krbvqvw2ag6a8bjx9c"; + libName = "k8s_openapi"; authors = [ - "clux " - "Natalie Klestrup Röijezon " - "kazk " + "Arnav Singh " ]; dependencies = [ { - name = "k8s-openapi"; - packageId = "k8s-openapi"; + name = "base64"; + packageId = "base64 0.22.1"; usesDefaultFeatures = false; + features = [ "alloc" ]; } { - name = "kube-client"; - packageId = "kube-client"; - optional = true; + name = "chrono"; + packageId = "chrono"; usesDefaultFeatures = false; + features = [ "alloc" "serde" ]; } { - name = "kube-core"; - packageId = "kube-core"; + name = "schemars"; + packageId = "schemars"; + optional = true; + usesDefaultFeatures = false; } { - name = "kube-derive"; - packageId = "kube-derive"; - optional = true; + name = "serde"; + packageId = "serde"; + usesDefaultFeatures = false; } { - name = "kube-runtime"; - packageId = "kube-runtime"; - optional = true; + name = "serde-value"; + packageId = "serde-value"; + usesDefaultFeatures = false; } - ]; - devDependencies = [ { - name = "k8s-openapi"; - packageId = "k8s-openapi"; + name = "serde_json"; + packageId = "serde_json"; usesDefaultFeatures = false; - features = [ "latest" ]; + features = [ "alloc" ]; } ]; features = { - "admission" = [ "kube-core/admission" ]; - "aws-lc-rs" = [ "kube-client?/aws-lc-rs" ]; - "client" = [ "kube-client/client" "config" ]; - "config" = [ "kube-client/config" ]; + "earliest" = [ "v1_28" ]; + "latest" = [ "v1_32" ]; + "schemars" = [ "dep:schemars" ]; + }; + resolvedDefaultFeatures = [ "schemars" "v1_32" ]; + }; + "k8s-version" = rec { + crateName = "k8s-version"; + version = "0.1.2"; + edition = "2021"; + workspace_member = null; + src = pkgs.fetchgit { + url = "https://github.com/stackabletech/operator-rs.git"; + rev = "048c7d8befddc2f2c6414444006871c95412d67c"; + sha256 = "1x2pfibrsysmkkmajyj30qkwsjf3rzmc3dxsd09jb9r4x7va6mr6"; + }; + libName = "k8s_version"; + authors = [ + "Stackable GmbH " + ]; + dependencies = [ + { + name = "darling"; + packageId = "darling"; + optional = true; + } + { + name = "regex"; + packageId = "regex"; + } + { + name = "snafu"; + packageId = "snafu 0.8.5"; + } + ]; + features = { + "darling" = [ "dep:darling" ]; + }; + resolvedDefaultFeatures = [ "darling" ]; + }; + "kube 0.96.0" = rec { + crateName = "kube"; + version = "0.96.0"; + edition = "2021"; + sha256 = "07ws50li6nxja26b0w40k2dqir60k4s5fi2hsvjz6kmxy0yypzzg"; + authors = [ + "clux " + "Natalie Klestrup Röijezon " + "kazk " + ]; + dependencies = [ + { + name = "k8s-openapi"; + packageId = "k8s-openapi 0.23.0"; + usesDefaultFeatures = false; + } + { + name = "kube-client"; + packageId = "kube-client 0.96.0"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "kube-core"; + packageId = "kube-core 0.96.0"; + } + { + name = "kube-derive"; + packageId = "kube-derive 0.96.0"; + optional = true; + } + { + name = "kube-runtime"; + packageId = "kube-runtime 0.96.0"; + optional = true; + } + ]; + devDependencies = [ + { + name = "k8s-openapi"; + packageId = "k8s-openapi 0.23.0"; + usesDefaultFeatures = false; + features = [ "latest" ]; + } + ]; + features = { + "admission" = [ "kube-core/admission" ]; + "aws-lc-rs" = [ "kube-client?/aws-lc-rs" ]; + "client" = [ "kube-client/client" "config" ]; + "config" = [ "kube-client/config" ]; "default" = [ "client" "rustls-tls" ]; "derive" = [ "kube-derive" "kube-core/schema" ]; "gzip" = [ "kube-client/gzip" "client" ]; @@ -3947,11 +4269,83 @@ rec { }; resolvedDefaultFeatures = [ "client" "config" "derive" "jsonpatch" "kube-client" "kube-derive" "kube-runtime" "runtime" "rustls-tls" ]; }; - "kube-client" = rec { - crateName = "kube-client"; + "kube 0.98.0" = rec { + crateName = "kube"; version = "0.98.0"; edition = "2021"; - sha256 = "1jd06xwhnmzrzqrfwq7jlmmxl7dvaygmchjx363zmlgvrlwasd4x"; + sha256 = "1fiwllwzsvl7921k85c10d1nwjpg09ycqcvvihc4vbggjp23s19j"; + authors = [ + "clux " + "Natalie Klestrup Röijezon " + "kazk " + ]; + dependencies = [ + { + name = "k8s-openapi"; + packageId = "k8s-openapi 0.24.0"; + usesDefaultFeatures = false; + } + { + name = "kube-client"; + packageId = "kube-client 0.98.0"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "kube-core"; + packageId = "kube-core 0.98.0"; + } + { + name = "kube-derive"; + packageId = "kube-derive 0.98.0"; + optional = true; + } + { + name = "kube-runtime"; + packageId = "kube-runtime 0.98.0"; + optional = true; + } + ]; + devDependencies = [ + { + name = "k8s-openapi"; + packageId = "k8s-openapi 0.24.0"; + usesDefaultFeatures = false; + features = [ "latest" ]; + } + ]; + features = { + "admission" = [ "kube-core/admission" ]; + "aws-lc-rs" = [ "kube-client?/aws-lc-rs" ]; + "client" = [ "kube-client/client" "config" ]; + "config" = [ "kube-client/config" ]; + "default" = [ "client" "rustls-tls" ]; + "derive" = [ "kube-derive" "kube-core/schema" ]; + "gzip" = [ "kube-client/gzip" "client" ]; + "http-proxy" = [ "kube-client/http-proxy" "client" ]; + "jsonpatch" = [ "kube-core/jsonpatch" ]; + "kube-client" = [ "dep:kube-client" ]; + "kube-derive" = [ "dep:kube-derive" ]; + "kube-runtime" = [ "dep:kube-runtime" ]; + "kubelet-debug" = [ "kube-client/kubelet-debug" "kube-core/kubelet-debug" ]; + "oauth" = [ "kube-client/oauth" "client" ]; + "oidc" = [ "kube-client/oidc" "client" ]; + "openssl-tls" = [ "kube-client/openssl-tls" "client" ]; + "runtime" = [ "kube-runtime" ]; + "rustls-tls" = [ "kube-client/rustls-tls" "client" ]; + "socks5" = [ "kube-client/socks5" "client" ]; + "unstable-client" = [ "kube-client/unstable-client" "client" ]; + "unstable-runtime" = [ "kube-runtime/unstable-runtime" "runtime" ]; + "webpki-roots" = [ "kube-client/webpki-roots" "client" ]; + "ws" = [ "kube-client/ws" "kube-core/ws" ]; + }; + resolvedDefaultFeatures = [ "client" "config" "derive" "jsonpatch" "kube-client" "kube-derive" "kube-runtime" "runtime" "rustls-tls" ]; + }; + "kube-client 0.96.0" = rec { + crateName = "kube-client"; + version = "0.96.0"; + edition = "2021"; + sha256 = "1wg0blziqkfyfmmyn6l1fj6wp7qy156sr3g7birj93gzx3n73x4b"; libName = "kube_client"; authors = [ "clux " @@ -4038,17 +4432,17 @@ rec { } { name = "jsonpath-rust"; - packageId = "jsonpath-rust"; + packageId = "jsonpath-rust 0.5.1"; optional = true; } { name = "k8s-openapi"; - packageId = "k8s-openapi"; + packageId = "k8s-openapi 0.23.0"; usesDefaultFeatures = false; } { name = "kube-core"; - packageId = "kube-core"; + packageId = "kube-core 0.96.0"; } { name = "pem"; @@ -4086,7 +4480,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.11"; + packageId = "thiserror 1.0.69"; } { name = "tokio"; @@ -4126,14 +4520,9 @@ rec { usesDefaultFeatures = false; features = [ "async-await" ]; } - { - name = "hyper"; - packageId = "hyper"; - features = [ "server" ]; - } { name = "k8s-openapi"; - packageId = "k8s-openapi"; + packageId = "k8s-openapi 0.23.0"; usesDefaultFeatures = false; features = [ "latest" ]; } @@ -4194,12 +4583,12 @@ rec { }; resolvedDefaultFeatures = [ "__non_core" "base64" "bytes" "chrono" "client" "config" "either" "futures" "home" "http-body" "http-body-util" "hyper" "hyper-rustls" "hyper-timeout" "hyper-util" "jsonpatch" "jsonpath-rust" "pem" "rustls" "rustls-pemfile" "rustls-tls" "serde_yaml" "tokio" "tokio-util" "tower" "tower-http" "tracing" ]; }; - "kube-core" = rec { - crateName = "kube-core"; + "kube-client 0.98.0" = rec { + crateName = "kube-client"; version = "0.98.0"; edition = "2021"; - sha256 = "1wwnsn1wk7bd2jiv9iw8446j0bczagqv1lc4wy88l5wa505q7alp"; - libName = "kube_core"; + sha256 = "1jd06xwhnmzrzqrfwq7jlmmxl7dvaygmchjx363zmlgvrlwasd4x"; + libName = "kube_client"; authors = [ "clux " "Natalie Klestrup Röijezon " @@ -4207,42 +4596,546 @@ rec { ]; dependencies = [ { - name = "chrono"; - packageId = "chrono"; + name = "base64"; + packageId = "base64 0.22.1"; + optional = true; + } + { + name = "bytes"; + packageId = "bytes"; + optional = true; + } + { + name = "chrono"; + packageId = "chrono"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "either"; + packageId = "either"; + optional = true; + } + { + name = "futures"; + packageId = "futures 0.3.31"; + optional = true; + usesDefaultFeatures = false; + features = [ "std" ]; + } + { + name = "home"; + packageId = "home"; + optional = true; + } + { + name = "http"; + packageId = "http"; + } + { + name = "http-body"; + packageId = "http-body"; + optional = true; + } + { + name = "http-body-util"; + packageId = "http-body-util"; + optional = true; + } + { + name = "hyper"; + packageId = "hyper"; + optional = true; + features = [ "client" "http1" ]; + } + { + name = "hyper-http-proxy"; + packageId = "hyper-http-proxy"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "hyper-rustls"; + packageId = "hyper-rustls"; + optional = true; + usesDefaultFeatures = false; + features = [ "http1" "logging" "native-tokio" "ring" "tls12" ]; + } + { + name = "hyper-timeout"; + packageId = "hyper-timeout"; + optional = true; + } + { + name = "hyper-util"; + packageId = "hyper-util"; + optional = true; + features = [ "client" "client-legacy" "http1" "tokio" ]; + } + { + name = "jsonpath-rust"; + packageId = "jsonpath-rust 0.7.5"; + optional = true; + } + { + name = "k8s-openapi"; + packageId = "k8s-openapi 0.24.0"; + usesDefaultFeatures = false; + } + { + name = "kube-core"; + packageId = "kube-core 0.98.0"; + } + { + name = "pem"; + packageId = "pem"; + optional = true; + } + { + name = "rustls"; + packageId = "rustls"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "rustls-pemfile"; + packageId = "rustls-pemfile"; + optional = true; + } + { + name = "secrecy"; + packageId = "secrecy"; + } + { + name = "serde"; + packageId = "serde"; + features = [ "derive" ]; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + { + name = "serde_yaml"; + packageId = "serde_yaml"; + optional = true; + } + { + name = "thiserror"; + packageId = "thiserror 2.0.11"; + } + { + name = "tokio"; + packageId = "tokio"; + optional = true; + features = [ "time" "signal" "sync" ]; + } + { + name = "tokio-util"; + packageId = "tokio-util"; + optional = true; + features = [ "io" "codec" ]; + } + { + name = "tower"; + packageId = "tower"; + optional = true; + features = [ "buffer" "filter" "util" ]; + } + { + name = "tower-http"; + packageId = "tower-http"; + optional = true; + features = [ "auth" "map-response-body" "trace" ]; + } + { + name = "tracing"; + packageId = "tracing"; + optional = true; + features = [ "log" ]; + } + ]; + devDependencies = [ + { + name = "futures"; + packageId = "futures 0.3.31"; + usesDefaultFeatures = false; + features = [ "async-await" ]; + } + { + name = "hyper"; + packageId = "hyper"; + features = [ "server" ]; + } + { + name = "k8s-openapi"; + packageId = "k8s-openapi 0.24.0"; + usesDefaultFeatures = false; + features = [ "latest" ]; + } + { + name = "tokio"; + packageId = "tokio"; + features = [ "full" ]; + } + ]; + features = { + "__non_core" = [ "tracing" "serde_yaml" "base64" ]; + "admission" = [ "kube-core/admission" ]; + "aws-lc-rs" = [ "rustls?/aws-lc-rs" ]; + "base64" = [ "dep:base64" ]; + "bytes" = [ "dep:bytes" ]; + "chrono" = [ "dep:chrono" ]; + "client" = [ "config" "__non_core" "hyper" "hyper-util" "http-body" "http-body-util" "tower" "tower-http" "hyper-timeout" "chrono" "jsonpath-rust" "bytes" "futures" "tokio" "tokio-util" "either" ]; + "config" = [ "__non_core" "pem" "home" ]; + "default" = [ "client" ]; + "either" = [ "dep:either" ]; + "form_urlencoded" = [ "dep:form_urlencoded" ]; + "futures" = [ "dep:futures" ]; + "gzip" = [ "client" "tower-http/decompression-gzip" ]; + "home" = [ "dep:home" ]; + "http-body" = [ "dep:http-body" ]; + "http-body-util" = [ "dep:http-body-util" ]; + "http-proxy" = [ "hyper-http-proxy" ]; + "hyper" = [ "dep:hyper" ]; + "hyper-http-proxy" = [ "dep:hyper-http-proxy" ]; + "hyper-openssl" = [ "dep:hyper-openssl" ]; + "hyper-rustls" = [ "dep:hyper-rustls" ]; + "hyper-socks2" = [ "dep:hyper-socks2" ]; + "hyper-timeout" = [ "dep:hyper-timeout" ]; + "hyper-util" = [ "dep:hyper-util" ]; + "jsonpatch" = [ "kube-core/jsonpatch" ]; + "jsonpath-rust" = [ "dep:jsonpath-rust" ]; + "kubelet-debug" = [ "ws" "kube-core/kubelet-debug" ]; + "oauth" = [ "client" "tame-oauth" ]; + "oidc" = [ "client" "form_urlencoded" ]; + "openssl" = [ "dep:openssl" ]; + "openssl-tls" = [ "openssl" "hyper-openssl" ]; + "pem" = [ "dep:pem" ]; + "rand" = [ "dep:rand" ]; + "rustls" = [ "dep:rustls" ]; + "rustls-pemfile" = [ "dep:rustls-pemfile" ]; + "rustls-tls" = [ "rustls" "rustls-pemfile" "hyper-rustls" "hyper-http-proxy?/rustls-tls-native-roots" ]; + "serde_yaml" = [ "dep:serde_yaml" ]; + "socks5" = [ "hyper-socks2" ]; + "tame-oauth" = [ "dep:tame-oauth" ]; + "tokio" = [ "dep:tokio" ]; + "tokio-tungstenite" = [ "dep:tokio-tungstenite" ]; + "tokio-util" = [ "dep:tokio-util" ]; + "tower" = [ "dep:tower" ]; + "tower-http" = [ "dep:tower-http" ]; + "tracing" = [ "dep:tracing" ]; + "webpki-roots" = [ "hyper-rustls/webpki-roots" ]; + "ws" = [ "client" "tokio-tungstenite" "rand" "kube-core/ws" "tokio/macros" ]; + }; + resolvedDefaultFeatures = [ "__non_core" "base64" "bytes" "chrono" "client" "config" "either" "futures" "home" "http-body" "http-body-util" "hyper" "hyper-rustls" "hyper-timeout" "hyper-util" "jsonpatch" "jsonpath-rust" "pem" "rustls" "rustls-pemfile" "rustls-tls" "serde_yaml" "tokio" "tokio-util" "tower" "tower-http" "tracing" ]; + }; + "kube-core 0.96.0" = rec { + crateName = "kube-core"; + version = "0.96.0"; + edition = "2021"; + sha256 = "0xrxzqk7nbbymf7ycm02wshs6ynf3dlrnm2wvix1skdk1g9lc8zl"; + libName = "kube_core"; + authors = [ + "clux " + "Natalie Klestrup Röijezon " + "kazk " + ]; + dependencies = [ + { + name = "chrono"; + packageId = "chrono"; + usesDefaultFeatures = false; + features = [ "now" ]; + } + { + name = "form_urlencoded"; + packageId = "form_urlencoded"; + } + { + name = "http"; + packageId = "http"; + } + { + name = "json-patch"; + packageId = "json-patch 2.0.0"; + optional = true; + } + { + name = "k8s-openapi"; + packageId = "k8s-openapi 0.23.0"; + usesDefaultFeatures = false; + } + { + name = "schemars"; + packageId = "schemars"; + optional = true; + } + { + name = "serde"; + packageId = "serde"; + features = [ "derive" ]; + } + { + name = "serde-value"; + packageId = "serde-value"; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + { + name = "thiserror"; + packageId = "thiserror 1.0.69"; + } + ]; + devDependencies = [ + { + name = "k8s-openapi"; + packageId = "k8s-openapi 0.23.0"; + usesDefaultFeatures = false; + features = [ "latest" ]; + } + ]; + features = { + "admission" = [ "json-patch" ]; + "json-patch" = [ "dep:json-patch" ]; + "jsonpatch" = [ "json-patch" ]; + "kubelet-debug" = [ "ws" ]; + "schema" = [ "schemars" ]; + "schemars" = [ "dep:schemars" ]; + }; + resolvedDefaultFeatures = [ "json-patch" "jsonpatch" "schema" "schemars" ]; + }; + "kube-core 0.98.0" = rec { + crateName = "kube-core"; + version = "0.98.0"; + edition = "2021"; + sha256 = "1wwnsn1wk7bd2jiv9iw8446j0bczagqv1lc4wy88l5wa505q7alp"; + libName = "kube_core"; + authors = [ + "clux " + "Natalie Klestrup Röijezon " + "kazk " + ]; + dependencies = [ + { + name = "chrono"; + packageId = "chrono"; + usesDefaultFeatures = false; + features = [ "now" ]; + } + { + name = "form_urlencoded"; + packageId = "form_urlencoded"; + } + { + name = "http"; + packageId = "http"; + } + { + name = "json-patch"; + packageId = "json-patch 3.0.1"; + optional = true; + } + { + name = "k8s-openapi"; + packageId = "k8s-openapi 0.24.0"; + usesDefaultFeatures = false; + } + { + name = "schemars"; + packageId = "schemars"; + optional = true; + } + { + name = "serde"; + packageId = "serde"; + features = [ "derive" ]; + } + { + name = "serde-value"; + packageId = "serde-value"; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + { + name = "thiserror"; + packageId = "thiserror 2.0.11"; + } + ]; + devDependencies = [ + { + name = "k8s-openapi"; + packageId = "k8s-openapi 0.24.0"; + usesDefaultFeatures = false; + features = [ "latest" ]; + } + ]; + features = { + "admission" = [ "json-patch" ]; + "json-patch" = [ "dep:json-patch" ]; + "jsonpatch" = [ "json-patch" ]; + "kubelet-debug" = [ "ws" ]; + "schema" = [ "schemars" ]; + "schemars" = [ "dep:schemars" ]; + }; + resolvedDefaultFeatures = [ "json-patch" "jsonpatch" "schema" "schemars" ]; + }; + "kube-derive 0.96.0" = rec { + crateName = "kube-derive"; + version = "0.96.0"; + edition = "2021"; + sha256 = "1bc23sismxyyncsry902b2i2v0aifpxvgs3fdh9q412yrh24wdpr"; + procMacro = true; + libName = "kube_derive"; + authors = [ + "clux " + "Natalie Klestrup Röijezon " + "kazk " + ]; + dependencies = [ + { + name = "darling"; + packageId = "darling"; + } + { + name = "proc-macro2"; + packageId = "proc-macro2"; + } + { + name = "quote"; + packageId = "quote"; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + { + name = "syn"; + packageId = "syn 2.0.96"; + features = [ "extra-traits" ]; + } + ]; + + }; + "kube-derive 0.98.0" = rec { + crateName = "kube-derive"; + version = "0.98.0"; + edition = "2021"; + sha256 = "0n46p76pvm3plsnbm57c2j76r1i6hwslxsaj345pxdvn8255sx1p"; + procMacro = true; + libName = "kube_derive"; + authors = [ + "clux " + "Natalie Klestrup Röijezon " + "kazk " + ]; + dependencies = [ + { + name = "darling"; + packageId = "darling"; + } + { + name = "proc-macro2"; + packageId = "proc-macro2"; + } + { + name = "quote"; + packageId = "quote"; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + { + name = "syn"; + packageId = "syn 2.0.96"; + features = [ "extra-traits" ]; + } + ]; + + }; + "kube-runtime 0.96.0" = rec { + crateName = "kube-runtime"; + version = "0.96.0"; + edition = "2021"; + sha256 = "0i5xr5i9xf44fwih1pvypr35sq30pcw979m9sbqnb3m9zzvg3yyk"; + libName = "kube_runtime"; + authors = [ + "clux " + "Natalie Klestrup Röijezon " + "kazk " + ]; + dependencies = [ + { + name = "ahash"; + packageId = "ahash"; + } + { + name = "async-broadcast"; + packageId = "async-broadcast"; + } + { + name = "async-stream"; + packageId = "async-stream"; + } + { + name = "async-trait"; + packageId = "async-trait"; + } + { + name = "backoff"; + packageId = "backoff"; + } + { + name = "educe"; + packageId = "educe"; usesDefaultFeatures = false; - features = [ "now" ]; + features = [ "Clone" "Debug" "Hash" "PartialEq" ]; } { - name = "form_urlencoded"; - packageId = "form_urlencoded"; + name = "futures"; + packageId = "futures 0.3.31"; + usesDefaultFeatures = false; + features = [ "async-await" ]; } { - name = "http"; - packageId = "http"; + name = "hashbrown"; + packageId = "hashbrown 0.14.5"; } { name = "json-patch"; - packageId = "json-patch"; - optional = true; + packageId = "json-patch 2.0.0"; + } + { + name = "jsonptr"; + packageId = "jsonptr 0.4.7"; } { name = "k8s-openapi"; - packageId = "k8s-openapi"; + packageId = "k8s-openapi 0.23.0"; usesDefaultFeatures = false; } { - name = "schemars"; - packageId = "schemars"; - optional = true; + name = "kube-client"; + packageId = "kube-client 0.96.0"; + usesDefaultFeatures = false; + features = [ "jsonpatch" "client" ]; } { - name = "serde"; - packageId = "serde"; - features = [ "derive" ]; + name = "parking_lot"; + packageId = "parking_lot"; } { - name = "serde-value"; - packageId = "serde-value"; + name = "pin-project"; + packageId = "pin-project"; + } + { + name = "serde"; + packageId = "serde"; } { name = "serde_json"; @@ -4250,65 +5143,45 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.11"; + packageId = "thiserror 1.0.69"; } - ]; - devDependencies = [ { - name = "k8s-openapi"; - packageId = "k8s-openapi"; - usesDefaultFeatures = false; - features = [ "latest" ]; + name = "tokio"; + packageId = "tokio"; + features = [ "time" ]; } - ]; - features = { - "admission" = [ "json-patch" ]; - "json-patch" = [ "dep:json-patch" ]; - "jsonpatch" = [ "json-patch" ]; - "kubelet-debug" = [ "ws" ]; - "schema" = [ "schemars" ]; - "schemars" = [ "dep:schemars" ]; - }; - resolvedDefaultFeatures = [ "json-patch" "jsonpatch" "schema" "schemars" ]; - }; - "kube-derive" = rec { - crateName = "kube-derive"; - version = "0.98.0"; - edition = "2021"; - sha256 = "0n46p76pvm3plsnbm57c2j76r1i6hwslxsaj345pxdvn8255sx1p"; - procMacro = true; - libName = "kube_derive"; - authors = [ - "clux " - "Natalie Klestrup Röijezon " - "kazk " - ]; - dependencies = [ { - name = "darling"; - packageId = "darling"; + name = "tokio-util"; + packageId = "tokio-util"; + features = [ "time" ]; } { - name = "proc-macro2"; - packageId = "proc-macro2"; + name = "tracing"; + packageId = "tracing"; } + ]; + devDependencies = [ { - name = "quote"; - packageId = "quote"; + name = "k8s-openapi"; + packageId = "k8s-openapi 0.23.0"; + usesDefaultFeatures = false; + features = [ "latest" ]; } { name = "serde_json"; packageId = "serde_json"; } { - name = "syn"; - packageId = "syn 2.0.96"; - features = [ "extra-traits" ]; + name = "tokio"; + packageId = "tokio"; + features = [ "full" "test-util" ]; } ]; - + features = { + "unstable-runtime" = [ "unstable-runtime-subscribe" "unstable-runtime-stream-control" "unstable-runtime-reconcile-on" ]; + }; }; - "kube-runtime" = rec { + "kube-runtime 0.98.0" = rec { crateName = "kube-runtime"; version = "0.98.0"; edition = "2021"; @@ -4354,7 +5227,7 @@ rec { } { name = "hashbrown"; - packageId = "hashbrown"; + packageId = "hashbrown 0.15.2"; } { name = "hostname"; @@ -4362,20 +5235,20 @@ rec { } { name = "json-patch"; - packageId = "json-patch"; + packageId = "json-patch 3.0.1"; } { name = "jsonptr"; - packageId = "jsonptr"; + packageId = "jsonptr 0.6.3"; } { name = "k8s-openapi"; - packageId = "k8s-openapi"; + packageId = "k8s-openapi 0.24.0"; usesDefaultFeatures = false; } { name = "kube-client"; - packageId = "kube-client"; + packageId = "kube-client 0.98.0"; usesDefaultFeatures = false; features = [ "jsonpatch" "client" ]; } @@ -4417,7 +5290,7 @@ rec { devDependencies = [ { name = "k8s-openapi"; - packageId = "k8s-openapi"; + packageId = "k8s-openapi 0.24.0"; usesDefaultFeatures = false; features = [ "latest" ]; } @@ -4875,16 +5748,16 @@ rec { }; "openssl" = rec { crateName = "openssl"; - version = "0.10.68"; + version = "0.10.70"; edition = "2021"; - sha256 = "1xbiz2bmba2fibg70s462yk2fndp3f9vz11c7iw0ilh2y54bqx31"; + sha256 = "1ij21wa5hzip17v91gl9x3n4h0am10ivq065andqrfx8cvhv9kv1"; authors = [ "Steven Fackler " ]; dependencies = [ { name = "bitflags"; - packageId = "bitflags"; + packageId = "bitflags 2.8.0"; } { name = "cfg-if"; @@ -4956,10 +5829,10 @@ rec { }; "openssl-sys" = rec { crateName = "openssl-sys"; - version = "0.9.104"; + version = "0.9.105"; edition = "2021"; links = "openssl"; - sha256 = "0hf712xcxmycnlc09r8d446b3mwqchsbfrjv374fp7grrc3g7as5"; + sha256 = "1p59q259h73w58fgajyd588hzaj9r3vp3jy78xlqsnp09fwda8lb"; build = "build/main.rs"; libName = "openssl_sys"; authors = [ @@ -5904,7 +6777,7 @@ rec { dependencies = [ { name = "bitflags"; - packageId = "bitflags"; + packageId = "bitflags 2.8.0"; } ]; features = { @@ -6685,7 +7558,7 @@ rec { dependencies = [ { name = "bitflags"; - packageId = "bitflags"; + packageId = "bitflags 2.8.0"; } { name = "core-foundation"; @@ -6732,7 +7605,7 @@ rec { dependencies = [ { name = "bitflags"; - packageId = "bitflags"; + packageId = "bitflags 2.8.0"; } { name = "core-foundation"; @@ -7365,72 +8238,6 @@ rec { }; resolvedDefaultFeatures = [ "alloc" ]; }; - "stackable-druid-crd" = rec { - crateName = "stackable-druid-crd"; - version = "0.0.0-dev"; - edition = "2021"; - src = lib.cleanSourceWith { filter = sourceFilter; src = ./rust/crd; }; - libName = "stackable_druid_crd"; - authors = [ - "Stackable GmbH " - ]; - dependencies = [ - { - name = "indoc"; - packageId = "indoc"; - } - { - name = "product-config"; - packageId = "product-config"; - } - { - name = "semver"; - packageId = "semver"; - } - { - name = "serde"; - packageId = "serde"; - features = [ "derive" ]; - } - { - name = "serde_json"; - packageId = "serde_json"; - } - { - name = "snafu"; - packageId = "snafu 0.8.5"; - } - { - name = "stackable-operator"; - packageId = "stackable-operator"; - } - { - name = "strum"; - packageId = "strum"; - features = [ "derive" ]; - } - { - name = "tracing"; - packageId = "tracing"; - } - ]; - devDependencies = [ - { - name = "rstest"; - packageId = "rstest"; - } - { - name = "serde_yaml"; - packageId = "serde_yaml"; - } - { - name = "tokio"; - packageId = "tokio"; - features = [ "full" ]; - } - ]; - - }; "stackable-druid-operator" = rec { crateName = "stackable-druid-operator"; version = "0.0.0-dev"; @@ -7501,14 +8308,15 @@ rec { name = "snafu"; packageId = "snafu 0.8.5"; } - { - name = "stackable-druid-crd"; - packageId = "stackable-druid-crd"; - } { name = "stackable-operator"; packageId = "stackable-operator"; } + { + name = "stackable-versioned"; + packageId = "stackable-versioned"; + features = [ "k8s" ]; + } { name = "strum"; packageId = "strum"; @@ -7600,17 +8408,17 @@ rec { } { name = "json-patch"; - packageId = "json-patch"; + packageId = "json-patch 3.0.1"; } { name = "k8s-openapi"; - packageId = "k8s-openapi"; + packageId = "k8s-openapi 0.24.0"; usesDefaultFeatures = false; features = [ "schemars" "v1_32" ]; } { name = "kube"; - packageId = "kube"; + packageId = "kube 0.98.0"; usesDefaultFeatures = false; features = [ "client" "jsonpatch" "runtime" "derive" "rustls-tls" ]; } @@ -7755,7 +8563,7 @@ rec { dependencies = [ { name = "kube"; - packageId = "kube"; + packageId = "kube 0.98.0"; usesDefaultFeatures = false; features = [ "client" "jsonpatch" "runtime" "derive" "rustls-tls" ]; } @@ -7779,6 +8587,106 @@ rec { ]; }; + "stackable-versioned" = rec { + crateName = "stackable-versioned"; + version = "0.5.0"; + edition = "2021"; + workspace_member = null; + src = pkgs.fetchgit { + url = "https://github.com/stackabletech/operator-rs.git"; + rev = "048c7d8befddc2f2c6414444006871c95412d67c"; + sha256 = "1x2pfibrsysmkkmajyj30qkwsjf3rzmc3dxsd09jb9r4x7va6mr6"; + }; + libName = "stackable_versioned"; + authors = [ + "Stackable GmbH " + ]; + dependencies = [ + { + name = "stackable-versioned-macros"; + packageId = "stackable-versioned-macros"; + } + ]; + features = { + "full" = [ "k8s" ]; + "k8s" = [ "stackable-versioned-macros/k8s" ]; + }; + resolvedDefaultFeatures = [ "k8s" ]; + }; + "stackable-versioned-macros" = rec { + crateName = "stackable-versioned-macros"; + version = "0.5.0"; + edition = "2021"; + workspace_member = null; + src = pkgs.fetchgit { + url = "https://github.com/stackabletech/operator-rs.git"; + rev = "048c7d8befddc2f2c6414444006871c95412d67c"; + sha256 = "1x2pfibrsysmkkmajyj30qkwsjf3rzmc3dxsd09jb9r4x7va6mr6"; + }; + procMacro = true; + libName = "stackable_versioned_macros"; + authors = [ + "Stackable GmbH " + ]; + dependencies = [ + { + name = "convert_case"; + packageId = "convert_case"; + } + { + name = "darling"; + packageId = "darling"; + } + { + name = "itertools"; + packageId = "itertools"; + } + { + name = "k8s-openapi"; + packageId = "k8s-openapi 0.23.0"; + optional = true; + usesDefaultFeatures = false; + features = [ "schemars" "v1_31" ]; + } + { + name = "k8s-version"; + packageId = "k8s-version"; + features = [ "darling" ]; + } + { + name = "kube"; + packageId = "kube 0.96.0"; + optional = true; + usesDefaultFeatures = false; + features = [ "client" "jsonpatch" "runtime" "derive" "rustls-tls" ]; + } + { + name = "proc-macro2"; + packageId = "proc-macro2"; + } + { + name = "quote"; + packageId = "quote"; + } + { + name = "syn"; + packageId = "syn 2.0.96"; + } + ]; + devDependencies = [ + { + name = "k8s-openapi"; + packageId = "k8s-openapi 0.23.0"; + usesDefaultFeatures = false; + features = [ "schemars" "v1_31" ]; + } + ]; + features = { + "full" = [ "k8s" ]; + "k8s" = [ "dep:kube" "dep:k8s-openapi" ]; + }; + resolvedDefaultFeatures = [ "k8s" ]; + }; "strsim" = rec { crateName = "strsim"; version = "0.11.1"; @@ -8732,7 +9640,7 @@ rec { } { name = "bitflags"; - packageId = "bitflags"; + packageId = "bitflags 2.8.0"; } { name = "bytes"; @@ -9324,6 +10232,19 @@ rec { ]; }; + "unicode-segmentation" = rec { + crateName = "unicode-segmentation"; + version = "1.12.0"; + edition = "2018"; + sha256 = "14qla2jfx74yyb9ds3d2mpwpa4l4lzb9z57c6d2ba511458z5k7n"; + libName = "unicode_segmentation"; + authors = [ + "kwantam " + "Manish Goregaokar " + ]; + features = { + }; + }; "unicode-xid" = rec { crateName = "unicode-xid"; version = "0.2.6"; diff --git a/crate-hashes.json b/crate-hashes.json index 290d87f2..c7d32c3a 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -2,5 +2,8 @@ "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#stackable-operator-derive@0.3.1": "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5", "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#stackable-operator@0.85.0": "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5", "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#stackable-shared@0.0.1": "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-versioned-0.5.0#k8s-version@0.1.2": "1x2pfibrsysmkkmajyj30qkwsjf3rzmc3dxsd09jb9r4x7va6mr6", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-versioned-0.5.0#stackable-versioned-macros@0.5.0": "1x2pfibrsysmkkmajyj30qkwsjf3rzmc3dxsd09jb9r4x7va6mr6", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-versioned-0.5.0#stackable-versioned@0.5.0": "1x2pfibrsysmkkmajyj30qkwsjf3rzmc3dxsd09jb9r4x7va6mr6", "git+https://github.com/stackabletech/product-config.git?tag=0.7.0#product-config@0.7.0": "0gjsm80g6r75pm3824dcyiz4ysq1ka4c1if6k1mjm9cnd5ym0gny" } \ No newline at end of file