chore: Update to operator-rs 0.77.1 and use new S3 structs

Author: sbernauer

Cargo.lock

@@ -23,10 +23,10 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 serde_yaml = "0.9"
 snafu = "0.8"
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.74.0" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.77.1" }
 product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.7.0" }
 strum = { version = "0.26", features = ["derive"] }
-tokio = { version = "1.39", features = ["full"] }
+tokio = { version = "1.40", features = ["full"] }
 tracing = "0.1"
 
 # [patch."https://github.com/stackabletech/operator-rs.git"]

Cargo.nix

@@ -104,14 +104,14 @@ spec:
                             - reference
                       properties:
                         inline:
-                          description: Inline definition of an S3 connection.
+                          description: S3 connection definition as a resource. Learn more on the [S3 concept documentation](https://docs.stackable.tech/home/nightly/concepts/s3).
                           properties:
                             accessStyle:
+                              default: VirtualHosted
                               description: Which access style to use. Defaults to virtual hosted-style as most of the data products out there. Have a look at the [AWS documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html).
                               enum:
                                 - Path
                                 - VirtualHosted
-                              nullable: true
                               type: string
                             credentials:
                               description: If the S3 uses authentication you have to specify you S3 credentials. In the most cases a [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) providing `accessKey` and `secretKey` is sufficient.
@@ -121,6 +121,12 @@ spec:
                                   description: '[Scope](https://docs.stackable.tech/home/nightly/secret-operator/scope) of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass).'
                                   nullable: true
                                   properties:
+                                    listenerVolumes:
+                                      default: []
+                                      description: The listener volume scope allows Node and Service scopes to be inferred from the applicable listeners. This must correspond to Volume names in the Pod that mount Listeners.
+                                      items:
+                                        type: string
+                                      type: array
                                     node:
                                       default: false
                                       description: The node scope is resolved to the name of the Kubernetes Node object that the Pod is running on. This will typically be the DNS name of the node.
@@ -143,8 +149,7 @@ spec:
                                 - secretClass
                               type: object
                             host:
-                              description: 'Hostname of the S3 server without any protocol or port. For example: `west1.my-cloud.com`.'
-                              nullable: true
+                              description: 'Host of the S3 server without any protocol or port. For example: `west1.my-cloud.com`.'
                               type: string
                             port:
                               description: Port the S3 server listens on. If not specified the product will determine the port to use.
@@ -153,7 +158,7 @@ spec:
                               nullable: true
                               type: integer
                             tls:
-                              description: If you want to use TLS when talking to S3 you can enable TLS encrypted communication with this setting.
+                              description: Use a TLS connection. If not specified no TLS will be used.
                               nullable: true
                               properties:
                                 verification:
@@ -192,9 +197,10 @@ spec:
                               required:
                                 - verification
                               type: object
+                          required:
+                            - host
                           type: object
                         reference:
-                          description: A reference to an S3Connection resource.
                           type: string
                       type: object
                     vectorAggregatorConfigMapName:
@@ -251,8 +257,10 @@ spec:
                         description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace.
                         properties:
                           name:
-                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                            description: 'Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
                             type: string
+                        required:
+                          - name
                         type: object
                       nullable: true
                       type: array
@@ -306,10 +314,6 @@ spec:
                               nullable: true
                               type: object
                               x-kubernetes-preserve-unknown-fields: true
-                          required:
-                            - nodeAffinity
-                            - podAffinity
-                            - podAntiAffinity
                           type: object
                         gracefulShutdownTimeout:
                           description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details.
@@ -578,10 +582,6 @@ spec:
                                     nullable: true
                                     type: object
                                     x-kubernetes-preserve-unknown-fields: true
-                                required:
-                                  - nodeAffinity
-                                  - podAffinity
-                                  - podAntiAffinity
                                 type: object
                               gracefulShutdownTimeout:
                                 description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details.

Cargo.toml

@@ -13,9 +13,12 @@ use stackable_operator::{
             CpuLimitsFragment, MemoryLimitsFragment, NoRuntimeLimits, NoRuntimeLimitsFragment,
             PvcConfig, PvcConfigFragment, Resources, ResourcesFragment,
         },
-        s3::S3ConnectionDef,
+        s3::S3ConnectionInlineOrReference,
+    },
+    config::{
+        fragment::{self, Fragment, ValidationError},
+        merge::Merge,
     },
-    config::{fragment, fragment::Fragment, fragment::ValidationError, merge::Merge},
     k8s_openapi::apimachinery::pkg::api::resource::Quantity,
     kube::{runtime::reflector::ObjectRef, CustomResource, ResourceExt},
     product_config_utils::{self, Configuration},
@@ -150,7 +153,7 @@ pub struct HiveClusterConfig {
     /// S3 connection specification. This can be either `inline` or a `reference` to an
     /// S3Connection object. Read the [S3 concept documentation](DOCS_BASE_URL_PLACEHOLDER/concepts/s3) to learn more.
     #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub s3: Option<S3ConnectionDef>,
+    pub s3: Option<S3ConnectionInlineOrReference>,
 
     /// Name of the Vector aggregator [discovery ConfigMap](DOCS_BASE_URL_PLACEHOLDER/concepts/service_discovery).
     /// It must contain the key `ADDRESS` with the address of the Vector aggregator.

crate-hashes.json

@@ -4,16 +4,7 @@ use stackable_hive_crd::{
     STACKABLE_CONFIG_MOUNT_DIR, STACKABLE_LOG_CONFIG_MOUNT_DIR, STACKABLE_TRUST_STORE,
     STACKABLE_TRUST_STORE_PASSWORD, SYSTEM_TRUST_STORE, SYSTEM_TRUST_STORE_PASSWORD,
 };
-use stackable_operator::commons::{
-    authentication::tls::{CaCert, Tls, TlsServerVerification, TlsVerification},
-    s3::S3ConnectionSpec,
-};
-
-pub const S3_SECRET_DIR: &str = "/stackable/secrets";
-pub const S3_ACCESS_KEY: &str = "accessKey";
-pub const S3_SECRET_KEY: &str = "secretKey";
-pub const ACCESS_KEY_PLACEHOLDER: &str = "xxx_access_key_xxx";
-pub const SECRET_KEY_PLACEHOLDER: &str = "xxx_secret_key_xxx";
+use stackable_operator::commons::s3::S3ConnectionSpec;
 
 pub fn build_container_command_args(
     hive: &HiveCluster,
@@ -29,6 +20,10 @@ pub fn build_container_command_args(
         format!("echo copying {STACKABLE_LOG_CONFIG_MOUNT_DIR}/{HIVE_METASTORE_LOG4J2_PROPERTIES} to {STACKABLE_CONFIG_DIR}/{HIVE_METASTORE_LOG4J2_PROPERTIES}"),
         format!("cp -RL {STACKABLE_LOG_CONFIG_MOUNT_DIR}/{HIVE_METASTORE_LOG4J2_PROPERTIES} {STACKABLE_CONFIG_DIR}/{HIVE_METASTORE_LOG4J2_PROPERTIES}"),
 
+        // Template config files
+        format!("if test -f {STACKABLE_CONFIG_DIR}/core-site.xml; then config-utils template {STACKABLE_CONFIG_DIR}/core-site.xml; fi"),
+        format!("if test -f {STACKABLE_CONFIG_DIR}/hive-site.xml; then config-utils template {STACKABLE_CONFIG_DIR}/hive-site.xml; fi"),
+
         // Copy system truststore to stackable truststore
         format!("keytool -importkeystore -srckeystore {SYSTEM_TRUST_STORE} -srcstoretype jks -srcstorepass {SYSTEM_TRUST_STORE_PASSWORD} -destkeystore {STACKABLE_TRUST_STORE} -deststoretype pkcs12 -deststorepass {STACKABLE_TRUST_STORE_PASSWORD} -noprompt")
     ];
@@ -41,22 +36,9 @@ pub fn build_container_command_args(
     }
 
     if let Some(s3) = s3_connection_spec {
-        if s3.credentials.is_some() {
-            args.extend([
-                format!("echo replacing {ACCESS_KEY_PLACEHOLDER} and {SECRET_KEY_PLACEHOLDER} with secret values."),
-                format!("sed -i \"s|{ACCESS_KEY_PLACEHOLDER}|$(cat {S3_SECRET_DIR}/{S3_ACCESS_KEY})|g\" {STACKABLE_CONFIG_DIR}/{HIVE_SITE_XML}"),
-                format!("sed -i \"s|{SECRET_KEY_PLACEHOLDER}|$(cat {S3_SECRET_DIR}/{S3_SECRET_KEY})|g\" {STACKABLE_CONFIG_DIR}/{HIVE_SITE_XML}"),
-            ]);
-        }
-
-        if let Some(Tls {
-            verification:
-                TlsVerification::Server(TlsServerVerification {
-                    ca_cert: CaCert::SecretClass(secret_class),
-                }),
-        }) = &s3.tls
-        {
-            args.push(format!("keytool -importcert -file /stackable/certificates/{secret_class}-tls-certificate/ca.crt -alias stackable-{secret_class} -keystore {STACKABLE_TRUST_STORE} -storepass {STACKABLE_TRUST_STORE_PASSWORD} -noprompt"));
+        if let Some(ca_cert) = s3.tls.tls_ca_cert_mount_path() {
+            // The alias can not clash, as we only support a single S3Connection
+            args.push(format!("keytool -importcert -file {ca_cert} -alias stackable-s3-ca-cert -keystore {STACKABLE_TRUST_STORE} -storepass {STACKABLE_TRUST_STORE_PASSWORD} -noprompt"));
         }
     }