Skip to content

Commit 39e10d0

Browse files
SamBarkerSamBarker
authored
[FLINK-36031] [kubernetes-client] Migrate to Fabric8 interceptor API
1 parent 84c1934 commit 39e10d0

File tree

8 files changed

+853
-60
lines changed

8 files changed

+853
-60
lines changed

‎examples/flink-sql-runner-example/pom.xml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,6 @@ under the License.
5050
<version>${flink.version}</version>
5151
<scope>provided</scope>
5252
</dependency>
53-
5453
<!-- Add connector dependencies here. They must be in the default scope (compile). -->
5554

5655
<!-- Example:

‎flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/metrics/KubernetesClientMetrics.java

Lines changed: 104 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -18,22 +18,29 @@
1818

1919
package org.apache.flink.kubernetes.operator.metrics;
2020

21+
import org.apache.flink.annotation.VisibleForTesting;
2122
import org.apache.flink.kubernetes.operator.config.FlinkOperatorConfiguration;
2223
import org.apache.flink.kubernetes.operator.metrics.OperatorMetricUtils.SynchronizedMeterView;
2324
import org.apache.flink.metrics.Counter;
2425
import org.apache.flink.metrics.Histogram;
2526
import org.apache.flink.metrics.MeterView;
2627
import org.apache.flink.metrics.MetricGroup;
2728

28-
import okhttp3.Interceptor;
29-
import okhttp3.Request;
30-
import okhttp3.Response;
29+
import io.fabric8.kubernetes.client.http.AsyncBody;
30+
import io.fabric8.kubernetes.client.http.BasicBuilder;
31+
import io.fabric8.kubernetes.client.http.HttpRequest;
32+
import io.fabric8.kubernetes.client.http.HttpResponse;
33+
import io.fabric8.kubernetes.client.http.Interceptor;
34+
import org.slf4j.Logger;
35+
import org.slf4j.LoggerFactory;
3136

32-
import java.io.IOException;
37+
import java.nio.ByteBuffer;
3338
import java.util.ArrayList;
3439
import java.util.List;
3540
import java.util.Map;
41+
import java.util.concurrent.CompletableFuture;
3642
import java.util.concurrent.ConcurrentHashMap;
43+
import java.util.function.LongSupplier;
3744

3845
/** Kubernetes client metrics. */
3946
public class KubernetesClientMetrics implements Interceptor {
@@ -50,6 +57,7 @@ public class KubernetesClientMetrics implements Interceptor {
5057
public static final String COUNTER = "Count";
5158
public static final String METER = "NumPerSecond";
5259
public static final String HISTO = "TimeNanos";
60+
public static final String REQUEST_START_TIME_HEADER = "requestStartTimeNanos";
5361
private final Histogram responseLatency;
5462

5563
private final MetricGroup requestMetricGroup;
@@ -69,9 +77,20 @@ public class KubernetesClientMetrics implements Interceptor {
6977
private final Map<Integer, SynchronizedMeterView> responseCodeMeters =
7078
new ConcurrentHashMap<>();
7179
private final Map<String, Counter> requestMethodCounter = new ConcurrentHashMap<>();
80+
private final LongSupplier nanoTimeSource;
81+
82+
private final Logger logger = LoggerFactory.getLogger(KubernetesClientMetrics.class);
7283

7384
public KubernetesClientMetrics(
7485
MetricGroup parentGroup, FlinkOperatorConfiguration flinkOperatorConfiguration) {
86+
this(parentGroup, flinkOperatorConfiguration, System::nanoTime);
87+
}
88+
89+
public KubernetesClientMetrics(
90+
MetricGroup parentGroup,
91+
FlinkOperatorConfiguration flinkOperatorConfiguration,
92+
LongSupplier nanoTimeSource) {
93+
this.nanoTimeSource = nanoTimeSource;
7594
MetricGroup metricGroup = parentGroup.addGroup(KUBE_CLIENT_GROUP);
7695

7796
this.requestMetricGroup = metricGroup.addGroup(HTTP_REQUEST_GROUP);
@@ -121,29 +140,86 @@ public KubernetesClientMetrics(
121140
}
122141

123142
@Override
124-
public Response intercept(Chain chain) throws IOException {
125-
Request request = chain.request();
143+
public void before(BasicBuilder builder, HttpRequest request, RequestTags tags) {
144+
long requestStartTime = nanoTimeSource.getAsLong();
145+
// Attach a header to the request. We don't care if is actually sent or echoed back in the
146+
// response.
147+
// As the request is included in the after callbacks so we just read the value from the
148+
// headers on that.
149+
builder.setHeader(REQUEST_START_TIME_HEADER, String.valueOf(requestStartTime));
126150
updateRequestMetrics(request);
127-
Response response = null;
128-
final long startTime = System.nanoTime();
129-
try {
130-
response = chain.proceed(request);
131-
return response;
132-
} finally {
133-
updateResponseMetrics(response, startTime);
134-
}
135151
}
136152

137-
private void updateRequestMetrics(Request request) {
153+
@Override
154+
public void after(
155+
HttpRequest request,
156+
HttpResponse<?> response,
157+
AsyncBody.Consumer<List<ByteBuffer>> consumer) {
158+
trackRequestLatency(request);
159+
updateResponseMetrics(response);
160+
}
161+
162+
@Override
163+
public CompletableFuture<Boolean> afterFailure(
164+
BasicBuilder builder, HttpResponse<?> response, RequestTags tags) {
165+
this.requestFailedRateMeter.markEvent();
166+
return CompletableFuture.completedFuture(false);
167+
}
168+
169+
@Override
170+
public void afterConnectionFailure(HttpRequest request, Throwable failure) {
171+
trackRequestLatency(request);
172+
this.requestFailedRateMeter.markEvent();
173+
}
174+
175+
@VisibleForTesting
176+
Counter getRequestCounter() {
177+
return requestCounter;
178+
}
179+
180+
@VisibleForTesting
181+
Counter getResponseCounter() {
182+
return responseCounter;
183+
}
184+
185+
@VisibleForTesting
186+
Counter getRequestMethodCounter(String method) {
187+
return requestMethodCounter.get(method);
188+
}
189+
190+
@VisibleForTesting
191+
SynchronizedMeterView getRequestRateMeter() {
192+
return requestRateMeter;
193+
}
194+
195+
@VisibleForTesting
196+
SynchronizedMeterView getResponseCodeMeter(int statusCode) {
197+
return responseCodeMeters.get(statusCode);
198+
}
199+
200+
@VisibleForTesting
201+
List<SynchronizedMeterView> getResponseCodeGroupMeters() {
202+
return responseCodeGroupMeters;
203+
}
204+
205+
@VisibleForTesting
206+
Histogram getResponseLatency() {
207+
return responseLatency;
208+
}
209+
210+
@VisibleForTesting
211+
SynchronizedMeterView getRequestFailedRateMeter() {
212+
return requestFailedRateMeter;
213+
}
214+
215+
private void updateRequestMetrics(HttpRequest request) {
138216
this.requestRateMeter.markEvent();
139217
getCounterByRequestMethod(request.method()).inc();
140218
}
141219

142-
private void updateResponseMetrics(Response response, long startTimeNanos) {
143-
final long latency = System.nanoTime() - startTimeNanos;
220+
private void updateResponseMetrics(HttpResponse<?> response) {
144221
if (response != null) {
145222
this.responseRateMeter.markEvent();
146-
this.responseLatency.update(latency);
147223
getMeterViewByResponseCode(response.code()).markEvent();
148224
if (this.httpResponseCodeGroupsEnabled) {
149225
responseCodeGroupMeters.get(response.code() / 100 - 1).markEvent();
@@ -153,6 +229,16 @@ private void updateResponseMetrics(Response response, long startTimeNanos) {
153229
}
154230
}
155231

232+
private void trackRequestLatency(HttpRequest request) {
233+
final String header = request.header(REQUEST_START_TIME_HEADER);
234+
if (header != null) {
235+
final long currentNanos = nanoTimeSource.getAsLong();
236+
final long requestStartNanos = Long.parseLong(header);
237+
final long latency = currentNanos - requestStartNanos;
238+
this.responseLatency.update(latency);
239+
}
240+
}
241+
156242
private Counter getCounterByRequestMethod(String method) {
157243
return requestMethodCounter.computeIfAbsent(
158244
method,

‎flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/utils/KubernetesClientUtils.java

Lines changed: 7 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,6 @@
2727
import io.fabric8.kubernetes.client.Config;
2828
import io.fabric8.kubernetes.client.KubernetesClient;
2929
import io.fabric8.kubernetes.client.KubernetesClientBuilder;
30-
import io.fabric8.kubernetes.client.okhttp.OkHttpClientFactory;
31-
import okhttp3.OkHttpClient;
3230
import org.slf4j.Logger;
3331
import org.slf4j.LoggerFactory;
3432

@@ -37,6 +35,8 @@ public class KubernetesClientUtils {
3735

3836
private static final Logger LOG = LoggerFactory.getLogger(KubernetesClientUtils.class);
3937

38+
private static final String METRICS_INTERCEPTOR_NAME = "KubernetesClientMetrics";
39+
4040
public static KubernetesClient getKubernetesClient(
4141
FlinkOperatorConfiguration operatorConfig, MetricGroup metricGroup) {
4242
return getKubernetesClient(operatorConfig, metricGroup, null);
@@ -51,19 +51,11 @@ public static KubernetesClient getKubernetesClient(
5151
var clientBuilder = new KubernetesClientBuilder().withConfig(kubernetesClientConfig);
5252

5353
if (operatorConfig.isKubernetesClientMetricsEnabled()) {
54-
clientBuilder =
55-
clientBuilder.withHttpClientFactory(
56-
// This logic should be replaced with a more generic solution once the
57-
// fabric8 Interceptor class is improved to the point where this can be
58-
// implemented.
59-
new OkHttpClientFactory() {
60-
@Override
61-
protected void additionalConfig(OkHttpClient.Builder builder) {
62-
builder.addInterceptor(
63-
new KubernetesClientMetrics(
64-
metricGroup, operatorConfig));
65-
}
66-
});
54+
clientBuilder.withHttpClientBuilderConsumer(
55+
httpCLientBuilder ->
56+
httpCLientBuilder.addOrReplaceInterceptor(
57+
METRICS_INTERCEPTOR_NAME,
58+
new KubernetesClientMetrics(metricGroup, operatorConfig)));
6759
}
6860

6961
return clientBuilder.build();

‎flink-kubernetes-operator/src/main/resources/META-INF/NOTICE

Lines changed: 25 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -21,31 +21,31 @@ This project bundles the following dependencies under the Apache Software Licens
2121
- commons-cli:commons-cli:1.5.0
2222
- commons-collections:commons-collections:3.2.2
2323
- commons-io:commons-io:2.11.0
24-
- io.fabric8:kubernetes-client-api:jar:6.11.0
25-
- io.fabric8:kubernetes-client:jar:6.11.0
26-
- io.fabric8:kubernetes-httpclient-okhttp:jar:6.11.0
27-
- io.fabric8:kubernetes-model-admissionregistration:jar:6.11.0
28-
- io.fabric8:kubernetes-model-apiextensions:jar:6.11.0
29-
- io.fabric8:kubernetes-model-apps:jar:6.11.0
30-
- io.fabric8:kubernetes-model-autoscaling:jar:6.11.0
31-
- io.fabric8:kubernetes-model-batch:jar:6.11.0
32-
- io.fabric8:kubernetes-model-certificates:jar:6.11.0
33-
- io.fabric8:kubernetes-model-common:jar:6.11.0
34-
- io.fabric8:kubernetes-model-coordination:jar:6.11.0
35-
- io.fabric8:kubernetes-model-core:jar:6.11.0
36-
- io.fabric8:kubernetes-model-discovery:jar:6.11.0
37-
- io.fabric8:kubernetes-model-events:jar:6.11.0
38-
- io.fabric8:kubernetes-model-extensions:jar:6.11.0
39-
- io.fabric8:kubernetes-model-flowcontrol:jar:6.11.0
40-
- io.fabric8:kubernetes-model-gatewayapi:jar:6.11.0
41-
- io.fabric8:kubernetes-model-metrics:jar:6.11.0
42-
- io.fabric8:kubernetes-model-networking:jar:6.11.0
43-
- io.fabric8:kubernetes-model-node:jar:6.11.0
44-
- io.fabric8:kubernetes-model-policy:jar:6.11.0
45-
- io.fabric8:kubernetes-model-rbac:jar:6.11.0
46-
- io.fabric8:kubernetes-model-resource:jar:6.11.0
47-
- io.fabric8:kubernetes-model-scheduling:jar:6.11.0
48-
- io.fabric8:kubernetes-model-storageclass:jar:6.11.0
24+
- io.fabric8:kubernetes-client-api:jar:6.13.2
25+
- io.fabric8:kubernetes-client:jar:6.13.2
26+
- io.fabric8:kubernetes-httpclient-okhttp:jar:6.13.2
27+
- io.fabric8:kubernetes-model-admissionregistration:jar:6.13.2
28+
- io.fabric8:kubernetes-model-apiextensions:jar:6.13.2
29+
- io.fabric8:kubernetes-model-apps:jar:6.13.2
30+
- io.fabric8:kubernetes-model-autoscaling:jar:6.13.2
31+
- io.fabric8:kubernetes-model-batch:jar:6.13.2
32+
- io.fabric8:kubernetes-model-certificates:jar:6.13.2
33+
- io.fabric8:kubernetes-model-common:jar:6.13.2
34+
- io.fabric8:kubernetes-model-coordination:jar:6.13.2
35+
- io.fabric8:kubernetes-model-core:jar:6.13.2
36+
- io.fabric8:kubernetes-model-discovery:jar:6.13.2
37+
- io.fabric8:kubernetes-model-events:jar:6.13.2
38+
- io.fabric8:kubernetes-model-extensions:jar:6.13.2
39+
- io.fabric8:kubernetes-model-flowcontrol:jar:6.13.2
40+
- io.fabric8:kubernetes-model-gatewayapi:jar:6.13.2
41+
- io.fabric8:kubernetes-model-metrics:jar:6.13.2
42+
- io.fabric8:kubernetes-model-networking:jar:6.13.2
43+
- io.fabric8:kubernetes-model-node:jar:6.13.2
44+
- io.fabric8:kubernetes-model-policy:jar:6.13.2
45+
- io.fabric8:kubernetes-model-rbac:jar:6.13.2
46+
- io.fabric8:kubernetes-model-resource:jar:6.13.2
47+
- io.fabric8:kubernetes-model-scheduling:jar:6.13.2
48+
- io.fabric8:kubernetes-model-storageclass:jar:6.13.2
4949
- io.fabric8:zjsonpatch:jar:0.3.0
5050
- io.javaoperatorsdk:operator-framework-core:jar:4.8.3
5151
- io.javaoperatorsdk:operator-framework:jar:4.8.3

0 commit comments

Comments
 (0)