[FLINK-36646][helm] Add required Java Opts to allow Java 17 based job images

.github/workflows/ci.yml

@@ -83,15 +83,33 @@ jobs:
       flink-version: "v1_20"
       http-client: ${{ matrix.http-client }}
       test: test_application_operations.sh
+  java_runtimes:
+    name: Java runtimes smoke test
+    needs: e2e_smoke_test
+    strategy:
+      matrix:
+        http-client: [ "okhttp" ]
+        java-version: [ "11", "17"]
+        flink-version:
+          - "v1_20"
+          - "v1_19"
+          - "v1_18"
+    uses: ./.github/workflows/e2e.yaml
+    with:
+      java-version: ${{ matrix.java-version }}
+      flink-version: ${{ matrix.flink-version }}
+      http-client: ${{ matrix.http-client }}
+      test: test_application_operations.sh
+      append-java-version: true
   e2e_namespace_tests:
     name: Alternative namespace tests
     needs: e2e_smoke_test
     strategy:
       matrix:
         flink-version:
           - "v1_20"
-          - "v1_18"
           - "v1_19"
+          - "v1_18"
           - "v1_17"
           - "v1_16"
         mode:

.github/workflows/e2e.yaml

@@ -27,6 +27,9 @@ on:
       create-namespace:
         type: boolean
         default: false
+      append-java-version:
+        type: boolean
+        default: false
 
 jobs:
   e2e_test:
@@ -78,6 +81,9 @@ jobs:
       - name: Run Flink e2e tests
         run: |
           FLINK_IMAGE=$(sed --regexp-extended 's/v([0-9]+)_([0-9]+)/flink:\1.\2/g' <<< ${{ inputs.flink-version }} )
+          if [[ "${{ inputs.append-java-version }}" == "true" ]]; then
+            FLINK_IMAGE=${FLINK_IMAGE}-java${{ inputs.java-version }}
+          fi
           echo FLINK_IMAGE=${FLINK_IMAGE}
           sed -i "s/image: flink:.*/image: ${FLINK_IMAGE}/" e2e-tests/data/*.yaml
           sed -i "s/flinkVersion: .*/flinkVersion: ${{ inputs.flink-version }}/" e2e-tests/data/*.yaml

docs/content/docs/operations/configuration.md

@@ -158,4 +158,33 @@ If you run Flink Operator in IPV6 environment, the [host name verification error
 due to a known bug in Okhttp client. As a workaround before new Okhttp 5.0.0 release, the environment variable below needs to be set 
 for both Flink Operator and Flink Deployment Configuration.
 
-KUBERNETES_DISABLE_HOSTNAME_VERIFICATION=true
+KUBERNETES_DISABLE_HOSTNAME_VERIFICATION=true
+
+## Java options for Java 17 based Job/Task Manager images
+
+### Summary
+
+From Java 17 onwards, Flink requires certain Java options to be supplied in order to function. These are set by default in the operator's `helm` configuration file. 
+However, users should note:
+
+- **Flink 1.19+**: If users want to change the default opts for all Flink deployments, they should **add to** rather than replace these default options (`kubernetes.operator.default-configuration.flink-version.<flink-version>.env.java.default-opts.all`) in the `helm` configuration. They should also avoid overriding `env.java.default-opts.all` in their `FlinkDeployment` configuration and use `env.java.opts.all` for any Java options needed by their own code.
+- **Flink 1.18**: Setting default Java opts is not available in Flink 1.18, so the operator sets `env.java.opts.all` to allow Java 17 to work. If users wish to use Flink 1.18 with Java 17 based images, they should avoid overriding `env.java.default.all` in their `FlinkDeployment` configuration. If custom Java opts are required, then they should include the [upstream Java options](https://github.com/apache/flink/blob/release-1.18.1/flink-dist/src/main/resources/flink-conf.yaml).
+
+### Details
+
+The default Job/Task Manager images use JRE 11 (Eclipse Temurin).
+However, there are official images published which use JRE 17 (e.g. `flink:1.20.0-scala_2.12-java17`) and users can customize the image used in their `FlinkDeployment` which can use a custom JRE.
+
+As Java 17 now requires the use of the module system ([Project Jigsaw](https://openjdk.org/projects/jigsaw/)), the user will need to supply the appropriate `add-exports` and `add-opens` Java options in order for the pre-module system code in Flink to function correctly with Java 17+.
+These options are defined upstream in the default configuration yaml (e.g. for [Flink 1.20](https://github.com/apache/flink/blob/release-1.20.0/flink-dist/src/main/resources/config.yaml)).
+
+Users wanting to run images based on Java 17 could supply a copy of these options by setting `env.java.opts.all`, in their `FlinkDeployment.spec.flinkconfiguration` to the upstream opts list and adding any options their code required.
+However, this has to be done for every deployment.
+To avoid this, the default configuration file used for `helm` sets the [`env.java.default-opts.all`](https://nightlies.apache.org/flink/flink-docs-master/docs/deployment/config/#env-java-default-opts-all) config to the upstream Flink Java options for the given Flink version.
+The default opts are appended to any user supplied options via `env.java.opts.all`.
+Therefore, users can continue to set Java options for their own code in their `FlinkDeployment`s and those required for Flink's core operations will be appended automatically.
+
+The exception to the above is Flink 1.18, as the `env.java.default-opts.all` option is not available in that version.
+For 1.18 the `helm` default config sets the `env.java.opts.all` directly.
+This will allow Java 17 based images to work correctly.
+However, if a user sets their own `env.java.opts.all` in their `FlinkDeployment`, then they will need to copy the [upstream Java options](https://github.com/apache/flink/blob/release-1.18.1/flink-dist/src/main/resources/flink-conf.yaml) into their list.

helm/flink-kubernetes-operator/conf/flink-conf.yaml

@@ -20,6 +20,14 @@
 taskmanager.numberOfTaskSlots: 1
 parallelism.default: 1
 
+# These parameters are required for Java 17 support.
+# These should be kept in-sync with the flink dist env.java.opts.all defaults (for the given flink version) in: flink-dist/src/main/resources/config.yaml
+# Flink 1.18 uses env.java.opts.all, if a user supplies their own version of these opts in their FlinkDeployment the options below will be overridden.
+# env.java.default-opts.all is used for 1.19 onwards so users can supply their own opts.all in their Job deployments and have these appended.
+kubernetes.operator.default-configuration.flink-version.v1_18.env.java.opts.all: --add-exports=java.base/sun.net.util=ALL-UNNAMED --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED
+kubernetes.operator.default-configuration.flink-version.v1_19.env.java.default-opts.all: --add-exports=java.base/sun.net.util=ALL-UNNAMED --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED
+kubernetes.operator.default-configuration.flink-version.v1_20.env.java.default-opts.all: --add-exports=java.base/sun.net.util=ALL-UNNAMED --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED
+
 # Flink operator related configs
 # kubernetes.operator.reconcile.interval: 60 s
 # kubernetes.operator.reconcile.parallelism: 5