[DEVX-468] Capture errors in telemetry middleware

commercetools/commercetools-monitoring-datadog/src/main/java/com/commercetools/monitoring/datadog/DatadogMiddleware.java

@@ -12,6 +12,7 @@
 import com.datadog.api.client.ApiException;
 import com.datadog.api.client.v2.api.MetricsApi;
 
+import io.vrap.rmf.base.client.ApiHttpException;
 import io.vrap.rmf.base.client.ApiHttpRequest;
 import io.vrap.rmf.base.client.ApiHttpResponse;
 import io.vrap.rmf.base.client.http.TelemetryMiddleware;
@@ -51,14 +52,23 @@
     public CompletableFuture<ApiHttpResponse<byte[]>> invoke(ApiHttpRequest request,
             Function<ApiHttpRequest, CompletableFuture<ApiHttpResponse<byte[]>>> next) {
         final Instant start = Instant.now();
-        return next.apply(request).thenApply(response -> {
+        return next.apply(request).handle((response, throwable) -> {
+            final int statusCode;
+            if (response != null) {
+                statusCode = response.getStatusCode();
+            }
+            else if (throwable instanceof ApiHttpException && ((ApiHttpException) throwable).getResponse() != null) {
+                statusCode = ((ApiHttpException) throwable).getResponse().getStatusCode();
+            }
+            else {
+                statusCode = 0;
+            }
             try {
                 submitClientDurationMetric(request, apiInstance, Duration.between(start, Instant.now()).toMillis(),
-                    response);
-                submitTotalRequestsMetric(request, apiInstance, response);
-
-                if (response.getStatusCode() >= 400) {
-                    submitErrorRequestsMetric(request, apiInstance, response);
+                    statusCode);
+                submitTotalRequestsMetric(request, apiInstance, statusCode);
+                if (statusCode >= 400 || throwable != null) {
+                    submitErrorRequestsMetric(request, apiInstance, statusCode);
                 }
             }
             catch (ApiException e) {

commercetools/commercetools-monitoring-datadog/src/main/java/com/commercetools/monitoring/datadog/DatadogUtils.java

@@ -28,29 +28,50 @@
 
     protected static void submitClientDurationMetric(final ApiHttpRequest request, final MetricsApi apiInstance,
             final double durationInMillis, final ApiHttpResponse<byte[]> response) throws ApiException {
+        submitClientDurationMetric(request, apiInstance, durationInMillis, response.getStatusCode());
+    }
+
+    protected static void submitClientDurationMetric(final ApiHttpRequest request, final MetricsApi apiInstance,
+            final double durationInMillis, final int statusCode) throws ApiException {
         final String name = PREFIX + "." + CLIENT_DURATION;
         final MetricIntakeType type = MetricIntakeType.UNSPECIFIED;
-        submitMetricWithHttpTags(name, durationInMillis, type, "ms", request, apiInstance, response);
+        submitMetricWithHttpTags(name, durationInMillis, type, "ms", request, apiInstance, statusCode);
     }
 
     protected static void submitErrorRequestsMetric(final ApiHttpRequest request, final MetricsApi apiInstance,
             final ApiHttpResponse<byte[]> response) throws ApiException {
+        submitErrorRequestsMetric(request, apiInstance, response.getStatusCode());
+    }
+
+    protected static void submitErrorRequestsMetric(final ApiHttpRequest request, final MetricsApi apiInstance,
+            final int statusCode) throws ApiException {
         final String name = PREFIX + "." + CLIENT_REQUEST_ERROR;
         final MetricIntakeType count = MetricIntakeType.COUNT;
-        submitMetricWithHttpTags(name, 1.0, count, "count", request, apiInstance, response);
+        submitMetricWithHttpTags(name, 1.0, count, "count", request, apiInstance, statusCode);
     }
 
     protected static void submitTotalRequestsMetric(final ApiHttpRequest request, final MetricsApi apiInstance,
             final ApiHttpResponse<byte[]> response) throws ApiException {
+        submitTotalRequestsMetric(request, apiInstance, response.getStatusCode());
+    }
+
+    protected static void submitTotalRequestsMetric(final ApiHttpRequest request, final MetricsApi apiInstance,
+            final int statusCode) throws ApiException {
         final String name = PREFIX + "." + CLIENT_REQUEST_TOTAL;
         final MetricIntakeType count = MetricIntakeType.COUNT;
-        submitMetricWithHttpTags(name, 1.0, count, "count", request, apiInstance, response);
+        submitMetricWithHttpTags(name, 1.0, count, "count", request, apiInstance, statusCode);
     }
 
     private static void submitMetricWithHttpTags(final String name, final double value, final MetricIntakeType type,
             final String unit, final ApiHttpRequest request, final MetricsApi apiInstance,
             final ApiHttpResponse<byte[]> response) throws ApiException {
-        final List<String> tags = Arrays.asList(format("%s:%s", HTTP_RESPONSE_STATUS_CODE, response.getStatusCode()),
+        submitMetricWithHttpTags(name, value, type, unit, request, apiInstance, response.getStatusCode());
+    }
+
+    private static void submitMetricWithHttpTags(final String name, final double value, final MetricIntakeType type,
+            final String unit, final ApiHttpRequest request, final MetricsApi apiInstance, final int statusCode)
+            throws ApiException {
+        final List<String> tags = Arrays.asList(format("%s:%s", HTTP_RESPONSE_STATUS_CODE, statusCode),
             format("%s:%s", HTTP_REQUEST_METHOD, request.getMethod().name()),
             format("%s:%s", SERVER_ADDRESS, request.getUri().getHost()));
         if (request.getUri().getPort() > 0) {

commercetools/commercetools-monitoring-datadog/src/main/java/com/commercetools/monitoring/datadog/statsd/DatadogMiddleware.java

@@ -13,6 +13,7 @@
 
 import com.timgroup.statsd.StatsDClient;
 
+import io.vrap.rmf.base.client.ApiHttpException;
 import io.vrap.rmf.base.client.ApiHttpRequest;
 import io.vrap.rmf.base.client.ApiHttpResponse;
 import io.vrap.rmf.base.client.http.TelemetryMiddleware;
@@ -49,9 +50,19 @@
             Function<ApiHttpRequest, CompletableFuture<ApiHttpResponse<byte[]>>> next) {
         final Instant start = Instant.now();
 
-        return next.apply(request).thenApply(response -> {
+        return next.apply(request).handle((response, throwable) -> {
             final List<String> tags = new ArrayList<>(4);
-            tags.add(format("%s:%s", HTTP_RESPONSE_STATUS_CODE, response.getStatusCode()));
+            final int statusCode;
+            if (response != null) {
+                statusCode = response.getStatusCode();
+            }
+            else if (throwable instanceof ApiHttpException && ((ApiHttpException) throwable).getResponse() != null) {
+                statusCode = ((ApiHttpException) throwable).getResponse().getStatusCode();
+            }
+            else {
+                statusCode = 0;
+            }
+            tags.add(format("%s:%s", HTTP_RESPONSE_STATUS_CODE, statusCode));
             tags.add(format("%s:%s", HTTP_REQUEST_METHOD, request.getMethod().name()));
             tags.add(format("%s:%s", SERVER_ADDRESS, request.getUri().getHost()));
             if (request.getUri().getPort() > 0) {
@@ -62,7 +73,7 @@
                 Duration.between(start, Instant.now()).toMillis(), tags.toArray(new String[0]));
 
             this.statsDClient.incrementCounter(PREFIX + "." + CLIENT_REQUEST_TOTAL, tags.toArray(new String[0]));
-            if (response.getStatusCode() >= 400) {
+            if (statusCode >= 400 || throwable != null) {
                 this.statsDClient.incrementCounter(PREFIX + "." + CLIENT_REQUEST_ERROR, tags.toArray(new String[0]));
             }
             return response;

commercetools/commercetools-monitoring-newrelic/src/main/java/com/commercetools/monitoring/newrelic/NewRelicTelemetryMiddleware.java

@@ -11,6 +11,7 @@
 
 import com.newrelic.api.agent.*;
 
+import io.vrap.rmf.base.client.ApiHttpException;
 import io.vrap.rmf.base.client.ApiHttpRequest;
 import io.vrap.rmf.base.client.ApiHttpResponse;
 import io.vrap.rmf.base.client.ContextApiHttpClientImpl;
@@ -61,21 +62,38 @@
         Optional<Token> token = context.map(NewRelicContext::getTransaction).map(Transaction::getToken);
         Optional<Segment> segment = context.map(c -> c.getTransaction()
                 .startSegment("commercetools", request.getMethod() + " " + request.getUri().getPath()));
-        return next.apply(request).thenApply(response -> {
+        return next.apply(request).handle((response, throwable) -> {
             token.ifPresent(Token::linkAndExpire);
+
+            final int statusCode;
+            final String message;
+            if (response != null) {
+                statusCode = response.getStatusCode();
+                message = response.getMessage();
+            }
+            else if (throwable instanceof ApiHttpException && ((ApiHttpException) throwable).getResponse() != null) {
+                ApiHttpResponse<byte[]> errorResponse = ((ApiHttpException) throwable).getResponse();
+                statusCode = errorResponse.getStatusCode();
+                message = errorResponse.getMessage();
+            }
+            else {
+                statusCode = 0;
+                message = throwable.getMessage();
+            }
+
             segment.ifPresent(s -> s.reportAsExternal(HttpParameters.library("commercetools-sdk-java-v2")
                     .uri(request.getUri())
                     .procedure(request.getMethod().name())
                     .noInboundHeaders()
-                    .status(response.getStatusCode(), response.getMessage())
+                    .status(statusCode, message)
                     .build()));
             segment.ifPresent(Segment::end);
 
             NewRelic.incrementCounter(PREFIX + CLIENT_REQUEST_TOTAL);
             NewRelic.recordResponseTimeMetric(PREFIX + CLIENT_DURATION,
                 Duration.between(start, Instant.now()).toMillis());
 
-            if (response.getStatusCode() >= 400) {
+            if (statusCode >= 400 || throwable != null) {
                 NewRelic.incrementCounter(PREFIX + CLIENT_REQUEST_ERROR);
             }
             return response;

commercetools/commercetools-monitoring-opentelemetry/src/main/java/com/commercetools/monitoring/opentelemetry/OpenTelemetryMiddleware.java

@@ -13,6 +13,7 @@
 import io.opentelemetry.api.metrics.LongCounter;
 import io.opentelemetry.api.metrics.LongHistogram;
 import io.opentelemetry.api.metrics.Meter;
+import io.vrap.rmf.base.client.ApiHttpException;
 import io.vrap.rmf.base.client.ApiHttpRequest;
 import io.vrap.rmf.base.client.ApiHttpResponse;
 import io.vrap.rmf.base.client.http.TelemetryMiddleware;
@@ -67,9 +68,19 @@
     public CompletableFuture<ApiHttpResponse<byte[]>> invoke(ApiHttpRequest request,
             Function<ApiHttpRequest, CompletableFuture<ApiHttpResponse<byte[]>>> next) {
         Instant start = Instant.now();
-        return next.apply(request).thenApply(response -> {
+        return next.apply(request).handle((response, throwable) -> {
+            final int statusCode;
+            if (response != null) {
+                statusCode = response.getStatusCode();
+            }
+            else if (throwable instanceof ApiHttpException && ((ApiHttpException) throwable).getResponse() != null) {
+                statusCode = ((ApiHttpException) throwable).getResponse().getStatusCode();
+            }
+            else {
+                statusCode = 0;
+            }
             AttributesBuilder builder = Attributes.builder()
-                    .put(OpenTelemetryInfo.HTTP_RESPONSE_STATUS_CODE, response.getStatusCode())
+                    .put(OpenTelemetryInfo.HTTP_RESPONSE_STATUS_CODE, statusCode)
                     .put(OpenTelemetryInfo.HTTP_REQUEST_METHOD, request.getMethod().name())
                     .put(OpenTelemetryInfo.SERVER_ADDRESS, request.getUri().getHost());
             if (request.getUri().getPort() > 0) {
@@ -79,7 +90,7 @@
             Optional.ofNullable(histogram)
                     .ifPresent(h -> h.record(Duration.between(start, Instant.now()).toMillis(), attributes));
             requestCounter.add(1, attributes);
-            if (response.getStatusCode() >= 400) {
+            if (statusCode >= 400 || throwable != null) {
                 errorCounter.add(1, attributes);
             }
             return response;