Skip to content

Commit 4f87bc2

Browse files
SamBarkerSamBarker
authored
[FLINK-36332] Add option to select the Fabric8 httpclient implementation
1 parent 70eb2b6 commit 4f87bc2

File tree

6 files changed

+105
-19
lines changed

6 files changed

+105
-19
lines changed

‎.github/workflows/ci.yml

Lines changed: 66 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,72 @@ jobs:
6767
cd flink-autoscaler-plugin-jdbc
6868
mvn -B verify -Dit.skip=false
6969
cd ..
70+
e2e_smoke_test:
71+
name: HTTP Client smoke test
72+
runs-on: ubuntu-latest
73+
strategy:
74+
matrix:
75+
http-client: [ "okhttp", "jdk", "jetty", "vertx" ]
76+
version: ["v1_20"]
77+
mode: ["native"]
78+
namespace: ["default"]
79+
java-version: ["21"]
80+
test:
81+
- test_application_operations.sh
82+
steps:
83+
- uses: actions/checkout@v4
84+
- name: Set up JDK ${{ matrix.java-version }}
85+
uses: actions/setup-java@v4
86+
with:
87+
java-version: ${{ matrix.java-version }}
88+
distribution: 'temurin'
89+
cache: 'maven'
90+
- name: Start minikube
91+
run: |
92+
source e2e-tests/utils.sh
93+
start_minikube
94+
- name: Install cert-manager
95+
run: |
96+
kubectl get pods -A
97+
kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.8.2/cert-manager.yaml
98+
kubectl -n cert-manager wait --all=true --for=condition=Available --timeout=300s deploy
99+
- name: Build image
100+
run: |
101+
export SHELL=/bin/bash
102+
export DOCKER_BUILDKIT=1
103+
eval $(minikube -p minikube docker-env)
104+
JAVA_VERSION=${{matrix.java-version}}
105+
HTTP_CLIENT=${{ matrix.http-client }}
106+
docker build --progress=plain --no-cache -f ./Dockerfile -t flink-kubernetes-operator:ci-latest --progress plain --build-arg JAVA_VERSION="${JAVA_VERSION:-11}" --build-arg HTTP_CLIENT="${HTTP_CLIENT:-okhttp}" .
107+
docker images
108+
- name: Start the operator
109+
run: |
110+
if [[ "${{ matrix.test }}" == "test_flink_operator_ha.sh" ]]; then
111+
sed -i "s/# kubernetes.operator.leader-election.enabled: false/kubernetes.operator.leader-election.enabled: true/" helm/flink-kubernetes-operator/conf/flink-conf.yaml
112+
sed -i "s/# kubernetes.operator.leader-election.lease-name: flink-operator-lease/kubernetes.operator.leader-election.lease-name: flink-operator-lease/" helm/flink-kubernetes-operator/conf/flink-conf.yaml
113+
sed -i "s/replicas: 1/replicas: 2/" helm/flink-kubernetes-operator/values.yaml
114+
fi
115+
helm --debug install flink-kubernetes-operator -n ${{ matrix.namespace }} helm/flink-kubernetes-operator --set image.repository=flink-kubernetes-operator --set image.tag=ci-latest ${{ matrix.extraArgs }}
116+
kubectl wait --for=condition=Available --timeout=120s -n ${{ matrix.namespace }} deploy/flink-kubernetes-operator
117+
kubectl get pods -n ${{ matrix.namespace }}
118+
- name: Run Flink e2e tests
119+
run: |
120+
sed -i "s/image: flink:.*/image: ${{ matrix.image }}/" e2e-tests/data/*.yaml
121+
sed -i "s/flinkVersion: .*/flinkVersion: ${{ matrix.version }}/" e2e-tests/data/*.yaml
122+
sed -i "s/mode: .*/mode: ${{ matrix.mode }}/" e2e-tests/data/*.yaml
123+
git diff HEAD
124+
echo "Running e2e-tests/$test"
125+
bash e2e-tests/${{ matrix.test }} || exit 1
126+
git reset --hard
127+
- name: Stop the operator
128+
run: |
129+
helm uninstall -n ${{ matrix.namespace }} flink-kubernetes-operator
130+
- name: Stop minikube
131+
run: |
132+
source e2e-tests/utils.sh
133+
stop_minikube
70134
e2e_ci:
135+
needs: e2e_smoke_test
71136
runs-on: ubuntu-latest
72137
strategy:
73138
matrix:
@@ -166,7 +231,7 @@ jobs:
166231
export SHELL=/bin/bash
167232
export DOCKER_BUILDKIT=1
168233
eval $(minikube -p minikube docker-env)
169-
export JAVA_VERSION=${{ matrix.java-version }}
234+
JAVA_VERSION=${{ matrix.java-version }}
170235
docker build --progress=plain --no-cache -f ./Dockerfile -t flink-kubernetes-operator:ci-latest --progress plain --build-arg JAVA_VERSION="${JAVA_VERSION:-11}" .
171236
docker images
172237
- name: Start the operator

‎Dockerfile

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,13 @@
1919
ARG JAVA_VERSION=11
2020
FROM maven:3.8.8-eclipse-temurin-${JAVA_VERSION} AS build
2121
ARG SKIP_TESTS=true
22+
ARG HTTP_CLIENT=okhttp
2223

2324
WORKDIR /app
2425

2526
COPY . .
2627

27-
RUN --mount=type=cache,target=/root/.m2 mvn -ntp clean install -pl flink-kubernetes-standalone,flink-kubernetes-operator-api,flink-kubernetes-operator,flink-autoscaler,flink-kubernetes-webhook -DskipTests=$SKIP_TESTS
28+
RUN --mount=type=cache,target=/root/.m2 mvn -ntp clean install -pl flink-kubernetes-standalone,flink-kubernetes-operator-api,flink-kubernetes-operator,flink-autoscaler,flink-kubernetes-webhook -DskipTests=$SKIP_TESTS -Dfabric8.httpclinent.impl="$HTTP_CLIENT"
2829

2930
RUN cd /app/tools/license; mkdir jars; cd jars; \
3031
cp /app/flink-kubernetes-operator/target/flink-kubernetes-operator-*-shaded.jar . && \

‎flink-kubernetes-operator/pom.xml

Lines changed: 33 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -38,13 +38,21 @@ under the License.
3838
<!-- required by FlinkConfigManagerTest -->
3939
--add-opens=java.base/java.util=ALL-UNNAMED
4040
</surefire.module.config>
41+
<!-- valid options can be checked at https://central.sonatype.com/search?q=kubernetes-httpclient- currently: okhttp, jdk, jetty, vertx -->
42+
<fabric8.httpclinent.impl>okhttp</fabric8.httpclinent.impl>
4143
</properties>
4244

4345
<dependencies>
4446
<dependency>
4547
<groupId>io.javaoperatorsdk</groupId>
4648
<artifactId>operator-framework</artifactId>
4749
<version>${operator.sdk.version}</version>
50+
<exclusions>
51+
<exclusion>
52+
<groupId>io.fabric8</groupId>
53+
<artifactId>kubernetes-httpclient-okhttp</artifactId>
54+
</exclusion>
55+
</exclusions>
4856
</dependency>
4957

5058
<dependency>
@@ -60,26 +68,17 @@ under the License.
6068
</dependency>
6169

6270
<dependency>
71+
<!-- https://github.com/fabric8io/kubernetes-client/blob/main/doc/FAQ.md#what-artifacts-should-my-project-depend-on-->
6372
<groupId>io.fabric8</groupId>
64-
<artifactId>kubernetes-httpclient-okhttp</artifactId>
73+
<artifactId>kubernetes-httpclient-${fabric8.httpclinent.impl}</artifactId>
6574
<version>${fabric8.version}</version>
6675
<exclusions>
6776
<exclusion>
6877
<groupId>com.squareup.okhttp3</groupId>
69-
<artifactId>*</artifactId>
78+
<artifactId>okhttp</artifactId>
7079
</exclusion>
7180
</exclusions>
7281
</dependency>
73-
<dependency>
74-
<groupId>com.squareup.okhttp3</groupId>
75-
<artifactId>okhttp</artifactId>
76-
<version>${okhttp.version}</version>
77-
</dependency>
78-
<dependency>
79-
<groupId>com.squareup.okhttp3</groupId>
80-
<artifactId>logging-interceptor</artifactId>
81-
<version>${okhttp.version}</version>
82-
</dependency>
8382

8483
<!-- Flink -->
8584
<dependency>
@@ -174,6 +173,12 @@ under the License.
174173
<artifactId>kubernetes-server-mock</artifactId>
175174
<version>${fabric8.version}</version>
176175
<scope>test</scope>
176+
<exclusions>
177+
<exclusion>
178+
<groupId>com.squareup.okhttp3</groupId>
179+
<artifactId>mockwebserver</artifactId>
180+
</exclusion>
181+
</exclusions>
177182
</dependency>
178183

179184
<dependency>
@@ -413,6 +418,22 @@ under the License.
413418
</plugins>
414419
</build>
415420
</profile>
421+
<profile>
422+
<id>depend-on-okhttp4</id>
423+
<activation>
424+
<property>
425+
<name>fabric8.httpclinent.impl</name>
426+
<value>okhttp</value>
427+
</property>
428+
</activation>
429+
<dependencies>
430+
<dependency>
431+
<groupId>com.squareup.okhttp3</groupId>
432+
<artifactId>okhttp</artifactId>
433+
<version>${okhttp.version}</version>
434+
</dependency>
435+
</dependencies>
436+
</profile>
416437
</profiles>
417438

418439
</project>

‎flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/state/KubernetesAutoScalerStateStore.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@
4040
import io.javaoperatorsdk.operator.processing.event.ResourceID;
4141
import lombok.SneakyThrows;
4242
import org.apache.commons.io.IOUtils;
43-
import org.jetbrains.annotations.NotNull;
4443
import org.slf4j.Logger;
4544
import org.slf4j.LoggerFactory;
4645

@@ -202,7 +201,7 @@ public Map<String, String> getParallelismOverrides(KubernetesJobAutoScalerContex
202201
.orElse(new HashMap<>());
203202
}
204203

205-
@NotNull
204+
@Nonnull
206205
@Override
207206
public ConfigChanges getConfigChanges(KubernetesJobAutoScalerContext jobContext) {
208207
return configMapStore

‎flink-kubernetes-operator/src/main/resources/META-INF/NOTICE

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,8 @@ This project bundles the following dependencies under the Apache Software Licens
4747
- io.fabric8:kubernetes-model-scheduling:jar:6.13.2
4848
- io.fabric8:kubernetes-model-storageclass:jar:6.13.2
4949
- io.fabric8:zjsonpatch:jar:0.3.0
50-
- io.javaoperatorsdk:operator-framework-core:jar:4.8.3
51-
- io.javaoperatorsdk:operator-framework:jar:4.8.3
50+
- io.javaoperatorsdk:operator-framework-core:jar:4.9.4
51+
- io.javaoperatorsdk:operator-framework:jar:4.9.4
5252
- org.apache.commons:commons-compress:1.21
5353
- org.apache.commons:commons-lang3:3.14.0
5454
- org.apache.commons:commons-math3:3.6.1

‎pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ under the License.
7575
<maven-javadoc-plugin.version>3.3.2</maven-javadoc-plugin.version>
7676
<git-commit-id-maven-plugin.version>5.0.0</git-commit-id-maven-plugin.version>
7777

78-
<operator.sdk.version>4.8.3</operator.sdk.version>
78+
<operator.sdk.version>4.9.4</operator.sdk.version>
7979
<operator.sdk.webhook-framework.version>1.1.1</operator.sdk.webhook-framework.version>
8080

8181
<fabric8.version>6.13.2</fabric8.version>

0 commit comments

Comments
 (0)