Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ env:
CARGO_TERM_COLOR: always
CARGO_INCREMENTAL: '0'
CARGO_PROFILE_DEV_DEBUG: '0'
RUST_TOOLCHAIN_VERSION: "1.79.0"
RUST_TOOLCHAIN_VERSION: "1.80.0"
RUSTFLAGS: "-D warnings"
RUSTDOCFLAGS: "-D warnings"
RUST_LOG: "info"
Expand Down
2 changes: 0 additions & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 0 additions & 8 deletions Cargo.nix

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ clap = "4.5"
fnv = "1.0"
futures = { version = "0.3", features = ["compat"] }
indoc = "2.0"
lazy_static = "1.5"
openssl = "0.10"
product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.7.0" }
pin-project = "1.1"
Expand Down
2 changes: 1 addition & 1 deletion rust-toolchain.toml
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
# DO NOT EDIT, this file is generated by operator-templating
[toolchain]
channel = "1.79.0"
channel = "1.80.0"
1 change: 0 additions & 1 deletion rust/crd/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ stackable-operator.workspace = true
strum.workspace = true
tracing.workspace = true
snafu.workspace = true
lazy_static.workspace = true

[dev-dependencies]
rstest.workspace = true
Expand Down
20 changes: 11 additions & 9 deletions rust/crd/src/memory.rs
Original file line number Diff line number Diff line change
@@ -1,24 +1,26 @@
use lazy_static::lazy_static;
use std::{collections::BTreeMap, sync::LazyLock};

use snafu::{OptionExt, ResultExt, Snafu};
use stackable_operator::{
commons::resources::{NoRuntimeLimits, Resources},
cpu::CpuQuantity,
memory::{BinaryMultiple, MemoryQuantity},
};
use std::collections::BTreeMap;

use crate::{
storage::HistoricalStorage, PROCESSING_BUFFER_SIZE_BYTES, PROCESSING_NUM_MERGE_BUFFERS,
PROCESSING_NUM_THREADS,
};

static MIN_HEAP_RATIO: f32 = 0.75;
lazy_static! {
pub static ref RESERVED_OS_MEMORY: MemoryQuantity = MemoryQuantity::from_mebi(300.);
/// Max size for direct access buffers. This is defined in Druid to be 2GB:
/// https://druid.apache.org/docs/latest/configuration/index.html#processing-1
static ref MAX_DIRECT_BUFFER_SIZE: MemoryQuantity = MemoryQuantity::from_gibi(2.);
}

pub static RESERVED_OS_MEMORY: LazyLock<MemoryQuantity> =
LazyLock::new(|| MemoryQuantity::from_mebi(300.));

/// Max size for direct access buffers. This is defined in Druid to be 2GB:
/// <https://druid.apache.org/docs/latest/configuration/index.html#processing-1>
pub static MAX_DIRECT_BUFFER_SIZE: LazyLock<MemoryQuantity> =
LazyLock::new(|| MemoryQuantity::from_gibi(2.));

#[derive(Snafu, Debug)]
pub enum Error {
Expand All @@ -41,7 +43,7 @@ pub enum Error {
/// [Druid Configuration Reference](https://druid.apache.org/docs/latest/configuration/index.html)
/// for additional information.
/// Also have a look at the documentation for
/// [Basic Cluster Tuning](<https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html>).
/// [Basic Cluster Tuning](https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html).
pub struct HistoricalDerivedSettings {
total_memory: MemoryQuantity,
cpu_millis: CpuQuantity,
Expand Down
135 changes: 70 additions & 65 deletions rust/crd/src/resource.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
use std::collections::BTreeMap;
use std::sync::LazyLock;

use crate::memory::{HistoricalDerivedSettings, RESERVED_OS_MEMORY};
use crate::storage::{self, default_free_percentage_empty_dir_fragment};
use crate::{DruidRole, PATH_SEGMENT_CACHE, PROP_SEGMENT_CACHE_LOCATIONS};
use lazy_static::lazy_static;
use snafu::{OptionExt, ResultExt, Snafu};
use stackable_operator::memory::MemoryQuantity;
use stackable_operator::{
Expand Down Expand Up @@ -150,70 +150,75 @@ impl RoleResource {
}
}

lazy_static! {
pub static ref MIDDLE_MANAGER_RESOURCES: ResourcesFragment<storage::DruidStorage, NoRuntimeLimits> =
ResourcesFragment {
cpu: CpuLimitsFragment {
min: Some(Quantity("300m".to_owned())),
max: Some(Quantity("1200m".to_owned())),
},
memory: MemoryLimitsFragment {
limit: Some(Quantity("1Gi".to_owned())),
runtime_limits: NoRuntimeLimitsFragment {},
},
storage: storage::DruidStorageFragment {},
};
pub static ref BROKER_RESOURCES: ResourcesFragment<storage::DruidStorage, NoRuntimeLimits> =
ResourcesFragment {
cpu: CpuLimitsFragment {
min: Some(Quantity("300m".to_owned())),
max: Some(Quantity("1200m".to_owned())),
},
memory: MemoryLimitsFragment {
limit: Some(Quantity("1500Mi".to_owned())),
runtime_limits: NoRuntimeLimitsFragment {},
},
storage: storage::DruidStorageFragment {},
};
pub static ref HISTORICAL_RESOURCES: ResourcesFragment<storage::HistoricalStorage, NoRuntimeLimits> =
ResourcesFragment {
cpu: CpuLimitsFragment {
min: Some(Quantity("300m".to_owned())),
max: Some(Quantity("1200m".to_owned())),
},
memory: MemoryLimitsFragment {
limit: Some(Quantity("1500Mi".to_owned())),
runtime_limits: NoRuntimeLimitsFragment {},
},
storage: storage::HistoricalStorageFragment {
segment_cache: default_free_percentage_empty_dir_fragment(),
},
};
pub static ref COORDINATOR_RESOURCES: ResourcesFragment<storage::DruidStorage, NoRuntimeLimits> =
ResourcesFragment {
cpu: CpuLimitsFragment {
min: Some(Quantity("100m".to_owned())),
max: Some(Quantity("400m".to_owned())),
},
memory: MemoryLimitsFragment {
limit: Some(Quantity("512Mi".to_owned())),
runtime_limits: NoRuntimeLimitsFragment {},
},
storage: storage::DruidStorageFragment {},
};
pub static ref ROUTER_RESOURCES: ResourcesFragment<storage::DruidStorage, NoRuntimeLimits> =
ResourcesFragment {
cpu: CpuLimitsFragment {
min: Some(Quantity("100m".to_owned())),
max: Some(Quantity("400m".to_owned())),
},
memory: MemoryLimitsFragment {
limit: Some(Quantity("512Mi".to_owned())),
runtime_limits: NoRuntimeLimitsFragment {},
},
storage: storage::DruidStorageFragment {},
};
}
pub static MIDDLE_MANAGER_RESOURCES: LazyLock<
ResourcesFragment<storage::DruidStorage, NoRuntimeLimits>,
> = LazyLock::new(|| ResourcesFragment {
cpu: CpuLimitsFragment {
min: Some(Quantity("300m".to_owned())),
max: Some(Quantity("1200m".to_owned())),
},
memory: MemoryLimitsFragment {
limit: Some(Quantity("1Gi".to_owned())),
runtime_limits: NoRuntimeLimitsFragment {},
},
storage: storage::DruidStorageFragment {},
});

pub static BROKER_RESOURCES: LazyLock<ResourcesFragment<storage::DruidStorage, NoRuntimeLimits>> =
LazyLock::new(|| ResourcesFragment {
cpu: CpuLimitsFragment {
min: Some(Quantity("300m".to_owned())),
max: Some(Quantity("1200m".to_owned())),
},
memory: MemoryLimitsFragment {
limit: Some(Quantity("1500Mi".to_owned())),
runtime_limits: NoRuntimeLimitsFragment {},
},
storage: storage::DruidStorageFragment {},
});

pub static HISTORICAL_RESOURCES: LazyLock<
ResourcesFragment<storage::HistoricalStorage, NoRuntimeLimits>,
> = LazyLock::new(|| ResourcesFragment {
cpu: CpuLimitsFragment {
min: Some(Quantity("300m".to_owned())),
max: Some(Quantity("1200m".to_owned())),
},
memory: MemoryLimitsFragment {
limit: Some(Quantity("1500Mi".to_owned())),
runtime_limits: NoRuntimeLimitsFragment {},
},
storage: storage::HistoricalStorageFragment {
segment_cache: default_free_percentage_empty_dir_fragment(),
},
});

pub static COORDINATOR_RESOURCES: LazyLock<
ResourcesFragment<storage::DruidStorage, NoRuntimeLimits>,
> = LazyLock::new(|| ResourcesFragment {
cpu: CpuLimitsFragment {
min: Some(Quantity("100m".to_owned())),
max: Some(Quantity("400m".to_owned())),
},
memory: MemoryLimitsFragment {
limit: Some(Quantity("512Mi".to_owned())),
runtime_limits: NoRuntimeLimitsFragment {},
},
storage: storage::DruidStorageFragment {},
});

pub static ROUTER_RESOURCES: LazyLock<ResourcesFragment<storage::DruidStorage, NoRuntimeLimits>> =
LazyLock::new(|| ResourcesFragment {
cpu: CpuLimitsFragment {
min: Some(Quantity("100m".to_owned())),
max: Some(Quantity("400m".to_owned())),
},
memory: MemoryLimitsFragment {
limit: Some(Quantity("512Mi".to_owned())),
runtime_limits: NoRuntimeLimitsFragment {},
},
storage: storage::DruidStorageFragment {},
});

#[cfg(test)]
mod test {
Expand Down
6 changes: 4 additions & 2 deletions rust/crd/src/security.rs
Original file line number Diff line number Diff line change
Expand Up @@ -119,10 +119,12 @@ impl DruidTlsSecurity {
}

/// Check if TLS encryption is enabled. This could be due to:
///
/// - A provided server `SecretClass`
/// - A provided client `AuthenticationClass` using tls
/// This affects init container commands, Druid configuration, volume mounts and
/// the Druid client port
///
/// This affects init container commands, Druid configuration, volume mounts
/// and the Druid client port
pub fn tls_enabled(&self) -> bool {
// TODO: This must be adapted if other authentication methods are supported and require TLS
self.auth_classes.tls_authentication_enabled()
Expand Down
1 change: 0 additions & 1 deletion rust/operator-binary/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ product-config.workspace = true
strum.workspace = true
tokio.workspace = true
tracing.workspace = true
lazy_static.workspace = true

[build-dependencies]
built.workspace = true
Expand Down
4 changes: 2 additions & 2 deletions rust/operator-binary/src/druid_controller.rs
Original file line number Diff line number Diff line change
Expand Up @@ -851,7 +851,7 @@ fn build_rolegroup_services(
Ok(Service {
metadata: ObjectMetaBuilder::new()
.name_and_namespace(druid)
.name(&rolegroup.object_name())
.name(rolegroup.object_name())
.ownerreference_from_resource(druid, None, Some(true))
.context(ObjectMissingMetadataForOwnerRefSnafu)?
.with_recommended_labels(build_recommended_labels(
Expand Down Expand Up @@ -1129,7 +1129,7 @@ fn build_rolegroup_statefulset(
Ok(StatefulSet {
metadata: ObjectMetaBuilder::new()
.name_and_namespace(druid)
.name(&rolegroup_ref.object_name())
.name(rolegroup_ref.object_name())
.ownerreference_from_resource(druid, None, Some(true))
.context(ObjectMissingMetadataForOwnerRefSnafu)?
.with_recommended_labels(build_recommended_labels(
Expand Down
Loading