From 235f5f4a7741ab77efdf6ea1ceb3a28f0f1071f2 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 21 Aug 2024 11:03:29 +0200 Subject: [PATCH 01/16] Starting with Superset-Operator --- rust/crd/src/lib.rs | 26 ++++++++++++++++++- .../src/superset_controller.rs | 9 ++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 94edcb13..00d6a13c 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -347,6 +347,28 @@ pub struct SupersetConfig { /// Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. #[fragment_attrs(serde(default))] pub graceful_shutdown_timeout: Option, + + #[fragment_attrs(serde(default))] + pub unsafe_python_customizations: PythonCustomizations, +} + +#[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)] +#[fragment_attrs( + derive( + Clone, + Debug, + Default, + Deserialize, + Merge, + JsonSchema, + PartialEq, + Serialize + ), + serde(rename_all = "camelCase") +)] +pub struct PythonCustomizations { + prefix: Option, + suffix: Option, } impl SupersetConfig { @@ -369,7 +391,9 @@ impl SupersetConfig { logging: product_logging::spec::default_logging(), affinity: get_affinity(cluster_name, role), graceful_shutdown_timeout: Some(DEFAULT_NODE_GRACEFUL_SHUTDOWN_TIMEOUT), - ..Default::default() + row_limit: None, + webserver_timeout: None, + unsafe_python_customizations: Default::default(), } } } diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index a559c7e3..8b568d5a 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -2,6 +2,7 @@ use std::{ borrow::Cow, collections::{BTreeMap, BTreeSet, HashMap}, + io::Write, sync::Arc, }; @@ -523,6 +524,10 @@ fn build_rolegroup_config_map( ); let mut config_file = Vec::new(); + + if let Some(prefix) = config_properties.remove("EXPERIMENTAL_FILE_HEADER") { + writeln!(config_file, "{}", prefix).unwrap(); + } flask_app_config_writer::write::( &mut config_file, config_properties.iter(), @@ -531,7 +536,9 @@ fn build_rolegroup_config_map( .with_context(|_| BuildRoleGroupConfigFileSnafu { rolegroup: rolegroup.clone(), })?; - + if let Some(prefix) = config_properties.remove("EXPERIMENTAL_FOOTER") { + writeln!(config_file, "{}", prefix).unwrap(); + } let mut cm_builder = ConfigMapBuilder::new(); cm_builder From b42df6ac16b01ef1fbfbcdd427532c84d105668c Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 21 Aug 2024 11:22:53 +0200 Subject: [PATCH 02/16] Add snafu error --- rust/crd/src/lib.rs | 23 ------------------- .../src/superset_controller.rs | 19 +++++++++++---- 2 files changed, 14 insertions(+), 28 deletions(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 00d6a13c..0cbf3284 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -347,28 +347,6 @@ pub struct SupersetConfig { /// Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. #[fragment_attrs(serde(default))] pub graceful_shutdown_timeout: Option, - - #[fragment_attrs(serde(default))] - pub unsafe_python_customizations: PythonCustomizations, -} - -#[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)] -#[fragment_attrs( - derive( - Clone, - Debug, - Default, - Deserialize, - Merge, - JsonSchema, - PartialEq, - Serialize - ), - serde(rename_all = "camelCase") -)] -pub struct PythonCustomizations { - prefix: Option, - suffix: Option, } impl SupersetConfig { @@ -393,7 +371,6 @@ impl SupersetConfig { graceful_shutdown_timeout: Some(DEFAULT_NODE_GRACEFUL_SHUTDOWN_TIMEOUT), row_limit: None, webserver_timeout: None, - unsafe_python_customizations: Default::default(), } } } diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index 8b568d5a..c2654537 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -240,7 +240,9 @@ pub enum Error { }, #[snafu(display("failed to add Superset config settings"))] - AddSupersetConfig { source: crate::config::Error }, + AddSupersetConfig { + source: crate::config::Error, + }, #[snafu(display("failed to add LDAP Volumes and VolumeMounts"))] AddLdapVolumesAndVolumeMounts { @@ -251,6 +253,10 @@ pub enum Error { AddTlsVolumesAndVolumeMounts { source: stackable_operator::commons::authentication::tls::TlsClientDetailsError, }, + + WriteToConfigFileString { + source: std::io::Error, + }, } type Result = std::result::Result; @@ -525,9 +531,10 @@ fn build_rolegroup_config_map( let mut config_file = Vec::new(); - if let Some(prefix) = config_properties.remove("EXPERIMENTAL_FILE_HEADER") { - writeln!(config_file, "{}", prefix).unwrap(); + if let Some(header) = config_properties.remove("EXPERIMENTAL_FILE_HEADER") { + writeln!(config_file, "{}", header).context(WriteToConfigFileStringSnafu)?; } + flask_app_config_writer::write::( &mut config_file, config_properties.iter(), @@ -536,9 +543,11 @@ fn build_rolegroup_config_map( .with_context(|_| BuildRoleGroupConfigFileSnafu { rolegroup: rolegroup.clone(), })?; - if let Some(prefix) = config_properties.remove("EXPERIMENTAL_FOOTER") { - writeln!(config_file, "{}", prefix).unwrap(); + + if let Some(footer) = config_properties.remove("EXPERIMENTAL_FILE_FOOTER") { + writeln!(config_file, "{}", footer).context(WriteToConfigFileStringSnafu)?; } + let mut cm_builder = ConfigMapBuilder::new(); cm_builder From 3f661eda1e806d8138ccf959b9c1b4d44d6ae45c Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 21 Aug 2024 11:25:46 +0200 Subject: [PATCH 03/16] Add snafu error message --- rust/operator-binary/src/superset_controller.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index c2654537..30a44e27 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -240,9 +240,7 @@ pub enum Error { }, #[snafu(display("failed to add Superset config settings"))] - AddSupersetConfig { - source: crate::config::Error, - }, + AddSupersetConfig { source: crate::config::Error }, #[snafu(display("failed to add LDAP Volumes and VolumeMounts"))] AddLdapVolumesAndVolumeMounts { @@ -254,9 +252,10 @@ pub enum Error { source: stackable_operator::commons::authentication::tls::TlsClientDetailsError, }, - WriteToConfigFileString { - source: std::io::Error, - }, + #[snafu(display( + "failed to write to String (Vec to be precise) containing superset config" + ))] + WriteToConfigFileString { source: std::io::Error }, } type Result = std::result::Result; From ab7e18b481b3fef17135dc641f1c8e112923c53b Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 21 Aug 2024 11:27:12 +0200 Subject: [PATCH 04/16] Add comment --- rust/operator-binary/src/superset_controller.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index 30a44e27..67120e05 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -530,6 +530,8 @@ fn build_rolegroup_config_map( let mut config_file = Vec::new(); + // By removing the key from `config_properties`, we avoid pasting the Python code into a Python variable as well + // (which would be bad) if let Some(header) = config_properties.remove("EXPERIMENTAL_FILE_HEADER") { writeln!(config_file, "{}", header).context(WriteToConfigFileStringSnafu)?; } @@ -543,6 +545,8 @@ fn build_rolegroup_config_map( rolegroup: rolegroup.clone(), })?; + // By removing the key from `config_properties`, we avoid pasting the Python code into a Python variable as well + // (which would be bad) if let Some(footer) = config_properties.remove("EXPERIMENTAL_FILE_FOOTER") { writeln!(config_file, "{}", footer).context(WriteToConfigFileStringSnafu)?; } From 76ea9d974921c36960ad8fd485f4c355a9542d4b Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 21 Aug 2024 11:58:19 +0200 Subject: [PATCH 05/16] Fixing footer variable --- rust/operator-binary/src/superset_controller.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index 67120e05..807dd86f 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -535,6 +535,7 @@ fn build_rolegroup_config_map( if let Some(header) = config_properties.remove("EXPERIMENTAL_FILE_HEADER") { writeln!(config_file, "{}", header).context(WriteToConfigFileStringSnafu)?; } + let temp_file_footer = config_properties.remove("EXPERIMENTAL_FILE_FOOTER"); flask_app_config_writer::write::( &mut config_file, @@ -547,7 +548,7 @@ fn build_rolegroup_config_map( // By removing the key from `config_properties`, we avoid pasting the Python code into a Python variable as well // (which would be bad) - if let Some(footer) = config_properties.remove("EXPERIMENTAL_FILE_FOOTER") { + if let Some(footer) = temp_file_footer { writeln!(config_file, "{}", footer).context(WriteToConfigFileStringSnafu)?; } From 4cf97524ca7e32008b321d5c82c2f76f21277eb1 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 21 Aug 2024 13:23:31 +0200 Subject: [PATCH 06/16] Adding comments on open todo to reference footer and header with vars from operator-rs --- rust/operator-binary/src/superset_controller.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index 807dd86f..8c533603 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -532,6 +532,7 @@ fn build_rolegroup_config_map( // By removing the key from `config_properties`, we avoid pasting the Python code into a Python variable as well // (which would be bad) + // TODO: Use public constants in operator-rs to reference `EXPERIMENTAL_FILE_HEADER` and `EXPERIMENTAL_FILE_FOOTER` if let Some(header) = config_properties.remove("EXPERIMENTAL_FILE_HEADER") { writeln!(config_file, "{}", header).context(WriteToConfigFileStringSnafu)?; } @@ -548,6 +549,7 @@ fn build_rolegroup_config_map( // By removing the key from `config_properties`, we avoid pasting the Python code into a Python variable as well // (which would be bad) + // TODO: Use public constants in operator-rs to reference `EXPERIMENTAL_FILE_HEADER` and `EXPERIMENTAL_FILE_FOOTER` if let Some(footer) = temp_file_footer { writeln!(config_file, "{}", footer).context(WriteToConfigFileStringSnafu)?; } From b3ff1895f90dd5e26bb701d9bce6f2f583d46b5a Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 21 Aug 2024 13:24:50 +0200 Subject: [PATCH 07/16] Comment about removing key and write later --- rust/operator-binary/src/superset_controller.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index 8c533603..c748afc4 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -536,6 +536,7 @@ fn build_rolegroup_config_map( if let Some(header) = config_properties.remove("EXPERIMENTAL_FILE_HEADER") { writeln!(config_file, "{}", header).context(WriteToConfigFileStringSnafu)?; } + // removing key from `config_properties` to avoid key value match. Append it later. let temp_file_footer = config_properties.remove("EXPERIMENTAL_FILE_FOOTER"); flask_app_config_writer::write::( From fb9b46a6bf764fcf67c16ae6a1a760486e86feb9 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 21 Aug 2024 14:59:28 +0200 Subject: [PATCH 08/16] Removing comments --- rust/operator-binary/src/superset_controller.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index c748afc4..fb855fe8 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -530,8 +530,6 @@ fn build_rolegroup_config_map( let mut config_file = Vec::new(); - // By removing the key from `config_properties`, we avoid pasting the Python code into a Python variable as well - // (which would be bad) // TODO: Use public constants in operator-rs to reference `EXPERIMENTAL_FILE_HEADER` and `EXPERIMENTAL_FILE_FOOTER` if let Some(header) = config_properties.remove("EXPERIMENTAL_FILE_HEADER") { writeln!(config_file, "{}", header).context(WriteToConfigFileStringSnafu)?; @@ -548,8 +546,6 @@ fn build_rolegroup_config_map( rolegroup: rolegroup.clone(), })?; - // By removing the key from `config_properties`, we avoid pasting the Python code into a Python variable as well - // (which would be bad) // TODO: Use public constants in operator-rs to reference `EXPERIMENTAL_FILE_HEADER` and `EXPERIMENTAL_FILE_FOOTER` if let Some(footer) = temp_file_footer { writeln!(config_file, "{}", footer).context(WriteToConfigFileStringSnafu)?; From 21c328bdb0b60bf7642c0fc9f3c735eff24b9ce6 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 22 Aug 2024 13:32:39 +0200 Subject: [PATCH 09/16] Bump operator-rs to 0.74.0, using constants rather then strings for header and footer keys --- Cargo.lock | 7 +++---- Cargo.toml | 2 +- rust/operator-binary/src/superset_controller.rs | 9 ++++++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 62b603bd..34558aea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2056,8 +2056,8 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "stackable-operator" -version = "0.73.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.73.0#4d98a29b08a7d959e5e287f774cf064c02ffbd62" +version = "0.74.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.74.0#c77a5423b66bc1667b63af7d8bec00de88a5303f" dependencies = [ "chrono", "clap", @@ -2070,7 +2070,6 @@ dependencies = [ "json-patch", "k8s-openapi", "kube", - "lazy_static", "opentelemetry-jaeger", "opentelemetry_sdk", "product-config", @@ -2094,7 +2093,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.73.0#4d98a29b08a7d959e5e287f774cf064c02ffbd62" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.74.0#c77a5423b66bc1667b63af7d8bec00de88a5303f" dependencies = [ "darling", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index 51d1f9d7..8ab7d385 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_yaml = "0.9" snafu = "0.8" -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.73.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.74.0" } strum = { version = "0.26", features = ["derive"] } tokio = { version = "1.39", features = ["full"] } tracing = "0.1" diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index fb855fe8..40fa737f 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -40,7 +40,10 @@ use stackable_operator::{ kube::{runtime::controller::Action, Resource, ResourceExt}, kvp::{Label, Labels}, logging::controller::ReconcilerError, - product_config_utils::{transform_all_roles_to_config, validate_all_roles_and_groups_config}, + product_config_utils::{ + transform_all_roles_to_config, validate_all_roles_and_groups_config, + CONFIG_OVERRIDE_FILE_FOOTER_KEY, CONFIG_OVERRIDE_FILE_HEADER_KEY, + }, product_logging::{ self, framework::{create_vector_shutdown_file_command, remove_vector_shutdown_file_command}, @@ -531,11 +534,11 @@ fn build_rolegroup_config_map( let mut config_file = Vec::new(); // TODO: Use public constants in operator-rs to reference `EXPERIMENTAL_FILE_HEADER` and `EXPERIMENTAL_FILE_FOOTER` - if let Some(header) = config_properties.remove("EXPERIMENTAL_FILE_HEADER") { + if let Some(header) = config_properties.remove(CONFIG_OVERRIDE_FILE_HEADER_KEY) { writeln!(config_file, "{}", header).context(WriteToConfigFileStringSnafu)?; } // removing key from `config_properties` to avoid key value match. Append it later. - let temp_file_footer = config_properties.remove("EXPERIMENTAL_FILE_FOOTER"); + let temp_file_footer = config_properties.remove(CONFIG_OVERRIDE_FILE_FOOTER_KEY); flask_app_config_writer::write::( &mut config_file, From 057039d9830443204dc422325d4a6d20e3387fc3 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 22 Aug 2024 13:34:04 +0200 Subject: [PATCH 10/16] Remove todo's --- rust/operator-binary/src/superset_controller.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index 40fa737f..b8f08215 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -533,7 +533,6 @@ fn build_rolegroup_config_map( let mut config_file = Vec::new(); - // TODO: Use public constants in operator-rs to reference `EXPERIMENTAL_FILE_HEADER` and `EXPERIMENTAL_FILE_FOOTER` if let Some(header) = config_properties.remove(CONFIG_OVERRIDE_FILE_HEADER_KEY) { writeln!(config_file, "{}", header).context(WriteToConfigFileStringSnafu)?; } @@ -549,7 +548,6 @@ fn build_rolegroup_config_map( rolegroup: rolegroup.clone(), })?; - // TODO: Use public constants in operator-rs to reference `EXPERIMENTAL_FILE_HEADER` and `EXPERIMENTAL_FILE_FOOTER` if let Some(footer) = temp_file_footer { writeln!(config_file, "{}", footer).context(WriteToConfigFileStringSnafu)?; } From cb4fc587e6df159d5281bee6e814634fe7998f89 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 22 Aug 2024 13:38:33 +0200 Subject: [PATCH 11/16] Updating changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc01942f..eb669d12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,13 +2,19 @@ ## [Unreleased] +### Added + +- Allowing arbitrary python code as EXPERIMENTAL_FILE_HEADER and EXPERIMENTAL_FILE_FOOTER in superset_config.py ([#530]) + ### Changed - Reduce CRD size from `472KB` to `45KB` by accepting arbitrary YAML input instead of the underlying schema for the following fields ([#528]): - `podOverrides` - `affinity` +- Bumping stackable-operator from 0.73.0 to 0.74.0 ([#530]) [#528]: https://github.com/stackabletech/superset-operator/pull/528 +[#530]: https://github.com/stackabletech/superset-operator/pull/530 ## [24.7.0] - 2024-07-24 From 1db84f16587ec0e163193cbe7d2c091bf6fe4a7b Mon Sep 17 00:00:00 2001 From: Maximilian Wittich <56642549+Maleware@users.noreply.github.com> Date: Thu, 22 Aug 2024 13:58:50 +0200 Subject: [PATCH 12/16] Apply suggestions from code review Co-authored-by: Sebastian Bernauer --- CHANGELOG.md | 1 - rust/operator-binary/src/superset_controller.rs | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb669d12..2cc4baba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,6 @@ - Reduce CRD size from `472KB` to `45KB` by accepting arbitrary YAML input instead of the underlying schema for the following fields ([#528]): - `podOverrides` - `affinity` -- Bumping stackable-operator from 0.73.0 to 0.74.0 ([#530]) [#528]: https://github.com/stackabletech/superset-operator/pull/528 [#530]: https://github.com/stackabletech/superset-operator/pull/530 diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index b8f08215..42abd6e6 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -533,10 +533,11 @@ fn build_rolegroup_config_map( let mut config_file = Vec::new(); + // By removing the keys from `config_properties`, we avoid pasting the Python code into a Python variable as well + // (which would be bad) if let Some(header) = config_properties.remove(CONFIG_OVERRIDE_FILE_HEADER_KEY) { writeln!(config_file, "{}", header).context(WriteToConfigFileStringSnafu)?; } - // removing key from `config_properties` to avoid key value match. Append it later. let temp_file_footer = config_properties.remove(CONFIG_OVERRIDE_FILE_FOOTER_KEY); flask_app_config_writer::write::( From 4b64920519cc82444e04536023f232d7cacade5f Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 22 Aug 2024 14:48:34 +0200 Subject: [PATCH 13/16] Adding docs --- .../configuration-environment-overrides.adoc | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/docs/modules/superset/pages/usage-guide/configuration-environment-overrides.adoc b/docs/modules/superset/pages/usage-guide/configuration-environment-overrides.adoc index 4594d753..6d8f2596 100644 --- a/docs/modules/superset/pages/usage-guide/configuration-environment-overrides.adoc +++ b/docs/modules/superset/pages/usage-guide/configuration-environment-overrides.adoc @@ -59,6 +59,33 @@ be taken to produce a valid configuration. For a full list of configuration options we refer to the https://github.com/apache/superset/blob/master/superset/config.py[main config file for Superset]. +As Superset can be configured with python code too, arbitrary code can be added to the `superset_conf.py`. +You can use either `EXPERIMENTAL_FILE_HEADER` to add code to the top or `EXPERIMENTAL_FILE_FOOTER` to add to the bottom. + +IMPORTANT: This is an experimental feature + +[source,yaml] +---- +nodes: + configOverrides: + superset_config.py: + CSV_EXPORT: "{'encoding': 'utf-8'}" + EXPERIMENTAL_FILE_HEADER: | + from modules.my_module import my_class + EXPERIMENTAL_FILE_FOOTER: | + import logging + from superset.security import SupersetSecurityManager + + class myCustomSecurityManger(SupersetSecurityManager): + def __init__(): + init() + + CUSTOM_SECURITY_MANAGER = myCustomSecurityManger + roleGroups: + default: + config: {} +---- + == Environment Variables In a similar fashion, environment variables can be (over)written. For example per role group: From b539dcf4fca4255fbeb54f56c62fcbdc3cc414d3 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Mon, 26 Aug 2024 10:56:32 +0200 Subject: [PATCH 14/16] Adding tests configs respect role and group level configs --- .../configuration-environment-overrides.adoc | 1 + .../kuttl/smoke/30-install-superset.yaml.j2 | 8 ++++++++ tests/templates/kuttl/smoke/32-assert.yaml | 12 ++++++++++++ 3 files changed, 21 insertions(+) create mode 100644 tests/templates/kuttl/smoke/32-assert.yaml diff --git a/docs/modules/superset/pages/usage-guide/configuration-environment-overrides.adoc b/docs/modules/superset/pages/usage-guide/configuration-environment-overrides.adoc index 6d8f2596..d8d6fa49 100644 --- a/docs/modules/superset/pages/usage-guide/configuration-environment-overrides.adoc +++ b/docs/modules/superset/pages/usage-guide/configuration-environment-overrides.adoc @@ -112,4 +112,5 @@ nodes: config: {} ---- + // cliOverrides don't make sense for this operator, so the feature is omitted for now diff --git a/tests/templates/kuttl/smoke/30-install-superset.yaml.j2 b/tests/templates/kuttl/smoke/30-install-superset.yaml.j2 index 7739c151..de203aa5 100644 --- a/tests/templates/kuttl/smoke/30-install-superset.yaml.j2 +++ b/tests/templates/kuttl/smoke/30-install-superset.yaml.j2 @@ -36,6 +36,14 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + configOverrides: + superset_config.py: + COMMON_CONF_VAR: '"role-value"' # overridden by role group below + ROLE_CONF_VAR: '"role-value"' # only defined here at role level roleGroups: default: + configOverrides: + superset_config.py: + COMMON_CONF_VAR: '"group-value"' # overrides role value + GROUP_CONF_VAR: '"group-value"' # only defined here at group level replicas: 1 diff --git a/tests/templates/kuttl/smoke/32-assert.yaml b/tests/templates/kuttl/smoke/32-assert.yaml new file mode 100644 index 00000000..122dab80 --- /dev/null +++ b/tests/templates/kuttl/smoke/32-assert.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +commands: + # + # Test confOverrides + # + - script: | + kubectl -n $NAMESPACE get cm superset-node-default -o yaml | yq -e '.data."superset_config.py"' | grep "COMMON_CONF_VAR = \"group-value\"" + kubectl -n $NAMESPACE get cm superset-node-default -o yaml | yq -e '.data."superset_config.py"' | grep "ROLE_CONF_VAR = \"role-value\"" + kubectl -n $NAMESPACE get cm superset-node-default -o yaml | yq -e '.data."superset_config.py"' | grep "GROUP_CONF_VAR = \"group-value\"" From d6986947e346c84ec1488eb455b3e8e8b117e897 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Mon, 26 Aug 2024 14:00:36 +0200 Subject: [PATCH 15/16] Revert: tests for conf overrides --- .../kuttl/smoke/30-install-superset.yaml.j2 | 8 -------- tests/templates/kuttl/smoke/32-assert.yaml | 12 ------------ 2 files changed, 20 deletions(-) delete mode 100644 tests/templates/kuttl/smoke/32-assert.yaml diff --git a/tests/templates/kuttl/smoke/30-install-superset.yaml.j2 b/tests/templates/kuttl/smoke/30-install-superset.yaml.j2 index de203aa5..7739c151 100644 --- a/tests/templates/kuttl/smoke/30-install-superset.yaml.j2 +++ b/tests/templates/kuttl/smoke/30-install-superset.yaml.j2 @@ -36,14 +36,6 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} - configOverrides: - superset_config.py: - COMMON_CONF_VAR: '"role-value"' # overridden by role group below - ROLE_CONF_VAR: '"role-value"' # only defined here at role level roleGroups: default: - configOverrides: - superset_config.py: - COMMON_CONF_VAR: '"group-value"' # overrides role value - GROUP_CONF_VAR: '"group-value"' # only defined here at group level replicas: 1 diff --git a/tests/templates/kuttl/smoke/32-assert.yaml b/tests/templates/kuttl/smoke/32-assert.yaml deleted file mode 100644 index 122dab80..00000000 --- a/tests/templates/kuttl/smoke/32-assert.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 600 -commands: - # - # Test confOverrides - # - - script: | - kubectl -n $NAMESPACE get cm superset-node-default -o yaml | yq -e '.data."superset_config.py"' | grep "COMMON_CONF_VAR = \"group-value\"" - kubectl -n $NAMESPACE get cm superset-node-default -o yaml | yq -e '.data."superset_config.py"' | grep "ROLE_CONF_VAR = \"role-value\"" - kubectl -n $NAMESPACE get cm superset-node-default -o yaml | yq -e '.data."superset_config.py"' | grep "GROUP_CONF_VAR = \"group-value\"" From eae88eed32b4cdea933e72501bd569f5468f24b2 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Mon, 26 Aug 2024 14:42:32 +0200 Subject: [PATCH 16/16] retrigger pipeline