diff --git a/.github/workflows/acceptance.yaml b/.github/workflows/acceptance.yaml index 732d9c2fcf1..ad370023f45 100644 --- a/.github/workflows/acceptance.yaml +++ b/.github/workflows/acceptance.yaml @@ -172,6 +172,21 @@ jobs: env: HIERO_MIRROR_IMPORTER_BLOCK_NODES_0_HOST: 'block-node-1.${SOLO_NAMESPACE}.svc.cluster.local' SPRING_PROFILES_ACTIVE: 'blocknode' + monitor: + env: + HIERO_MIRROR_MONITOR_HEALTH_RELEASE_FAILWHENINACTIVE: "true" + test: + annotations: + helm.sh/hook-delete-policy: before-hook-creation + enabled: true + rest: + test: + annotations: + helm.sh/hook-delete-policy: before-hook-creation + restjava: + test: + annotations: + helm.sh/hook-delete-policy: before-hook-creation test: cucumberTags: "@acceptance and not @ethereum and not @ethcall and not @crud" annotations: @@ -184,10 +199,28 @@ jobs: web3: env: HIERO_MIRROR_WEB3_OPCODE_TRACER_ENABLED: "true" + test: + annotations: + helm.sh/hook-delete-policy: before-hook-creation EOF else cat < mirror.yaml + monitor: + env: + HIERO_MIRROR_MONITOR_HEALTH_RELEASE_FAILWHENINACTIVE: "true" + test: + annotations: + helm.sh/hook-delete-policy: before-hook-creation + enabled: true + rest: + test: + annotations: + helm.sh/hook-delete-policy: before-hook-creation + restjava: + test: + annotations: + helm.sh/hook-delete-policy: before-hook-creation test: annotations: helm.sh/hook-delete-policy: before-hook-creation @@ -200,6 +233,9 @@ jobs: web3: env: HIERO_MIRROR_WEB3_OPCODE_TRACER_ENABLED: "true" + test: + annotations: + helm.sh/hook-delete-policy: before-hook-creation EOF fi @@ -215,7 +251,7 @@ jobs: - name: Run acceptance tests run: | helm test "${HELM_RELEASE_NAME}" -n "${SOLO_NAMESPACE}" --logs \ - --filter name="${HELM_RELEASE_NAME}-mirror-acceptance" --timeout 20m | tee output_log.txt + --timeout 20m | tee output_log.txt - name: Run k6 tests if: ${{ matrix.stream-type != 'BLOCK' }} diff --git a/charts/hedera-mirror-monitor/postman.json b/charts/hedera-mirror-monitor/postman.json new file mode 100644 index 00000000000..535996118eb --- /dev/null +++ b/charts/hedera-mirror-monitor/postman.json @@ -0,0 +1,44 @@ +{ + "info": { + "_postman_id": "e015390f-8007-4371-8703-0b9ad853f718", + "name": "Monitor API", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", + "_exporter_id": "25186531" + }, + "item": [ + { + "name": "Check Cluster Health", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('Status code is 200', function () {", + " pm.expect(pm.response.code).to.equal(200);", + "});" + ], + "type": "text/javascript", + "packages": {}, + "requests": {} + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/actuator/health/cluster", + "host": ["{{baseUrl}}"], + "path": ["actuator", "health", "cluster"] + } + }, + "response": [] + } + ], + "variable": [ + { + "key": "baseUrl", + "value": "http://localhost:8082" + } + ] +} diff --git a/charts/hedera-mirror-monitor/templates/tests/configmap.yaml b/charts/hedera-mirror-monitor/templates/tests/configmap.yaml new file mode 100644 index 00000000000..fdcd992377f --- /dev/null +++ b/charts/hedera-mirror-monitor/templates/tests/configmap.yaml @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: Apache-2.0 + +{{ if .Values.test.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + annotations: {{- toYaml .Values.test.annotations | nindent 4 }} + labels: {{- include "hedera-mirror-monitor.labels" . | nindent 4 }} + app.kubernetes.io/name: {{ include "hedera-mirror-monitor.name" . }}-test + name: {{ include "hedera-mirror-monitor.fullname" . }}-test + namespace: {{ include "hedera-mirror-monitor.namespace" . }} +data: + postman.json: |- + {{- .Values.test.postman | b64dec | default (.Files.Get "postman.json") | nindent 4 }} +{{- end -}} diff --git a/charts/hedera-mirror-monitor/templates/tests/pod.yaml b/charts/hedera-mirror-monitor/templates/tests/pod.yaml new file mode 100644 index 00000000000..8f975e2f141 --- /dev/null +++ b/charts/hedera-mirror-monitor/templates/tests/pod.yaml @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: Apache-2.0 + +{{ if .Values.test.enabled -}} +apiVersion: v1 +kind: Pod +metadata: + annotations: {{- toYaml .Values.test.annotations | nindent 4 }} + labels: {{- include "hedera-mirror-monitor.labels" . | nindent 4 }} + app.kubernetes.io/name: {{ include "hedera-mirror-monitor.name" . }}-test + name: {{ include "hedera-mirror-monitor.fullname" . }}-test + namespace: {{ include "hedera-mirror-monitor.namespace" . }} +spec: + containers: + - name: test + image: "{{ .Values.test.image.repository }}:{{ .Values.test.image.tag }}" + imagePullPolicy: {{ .Values.test.image.pullPolicy }} + args: + - run + - /test/postman.json + - --env-var + - baseUrl=http://{{ include "hedera-mirror-monitor.fullname" . }}:{{ .Values.service.port }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + volumeMounts: + - name: monitor-test + mountPath: /test + readOnly: true + imagePullSecrets: {{ toYaml .Values.test.image.pullSecrets | nindent 4 }} + priorityClassName: {{ .Values.test.priorityClassName }} + restartPolicy: Never + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + terminationGracePeriodSeconds: 1 + volumes: + - name: monitor-test + configMap: + defaultMode: 420 + name: {{ include "hedera-mirror-monitor.fullname" . }}-test +{{- end -}} diff --git a/charts/hedera-mirror-monitor/values.yaml b/charts/hedera-mirror-monitor/values.yaml index 7c600eb85dc..5dd945f525f 100644 --- a/charts/hedera-mirror-monitor/values.yaml +++ b/charts/hedera-mirror-monitor/values.yaml @@ -301,6 +301,20 @@ serviceMonitor: terminationGracePeriodSeconds: 60 +test: + annotations: + helm.sh/hook: test-success + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + helm.sh/hook-weight: "30" + enabled: false + image: + pullPolicy: IfNotPresent + pullSecrets: [] + repository: postman/newman + tag: 6.1.3-alpine + postman: "" # Custom postman.json in base64 encoding + priorityClassName: "" + tolerations: [] updateStrategy: diff --git a/docs/configuration.md b/docs/configuration.md index da3de83b209..457da018199 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -432,6 +432,7 @@ monitor. | ----------------------------------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | | `hiero.mirror.monitor.health.release.cacheExpiry` | 30s | The amount of time to cache cluster release health status | | `hiero.mirror.monitor.health.release.enabled` | false | Whether to enable cluster release health check | +| `hiero.mirror.monitor.health.release.failWhenInactive` | false | Whether to enable Status.DOWN when publisher or subscriber metric rates go to zero. | | `hiero.mirror.monitor.mirrorNode.grpc.host` | "" | The hostname of the mirror node's gRPC API | | `hiero.mirror.monitor.mirrorNode.grpc.port` | 5600 | The port of the mirror node's gRPC API | | `hiero.mirror.monitor.mirrorNode.rest.host` | "" | The hostname of the mirror node's REST API | diff --git a/monitor/src/main/java/org/hiero/mirror/monitor/health/ClusterHealthIndicator.java b/monitor/src/main/java/org/hiero/mirror/monitor/health/ClusterHealthIndicator.java index 7bede619487..a023eb86101 100644 --- a/monitor/src/main/java/org/hiero/mirror/monitor/health/ClusterHealthIndicator.java +++ b/monitor/src/main/java/org/hiero/mirror/monitor/health/ClusterHealthIndicator.java @@ -26,7 +26,9 @@ public class ClusterHealthIndicator implements ReactiveHealthIndicator { private static final Mono UNKNOWN = health(Status.UNKNOWN, "Publishing is inactive"); private static final Mono UP = health(Status.UP, ""); + private static final Mono DOWN = health(Status.DOWN, ""); + private final ReleaseHealthProperties releaseHealthProperties; private final MirrorSubscriber mirrorSubscriber; private final RestApiClient restApiClient; private final TransactionGenerator transactionGenerator; @@ -47,17 +49,18 @@ public Mono health() { : Mono.just(health)); } - // Returns unknown if all publish scenarios aggregated rate has dropped to zero, otherwise returns an empty flux + // Returns down or unknown if all publish scenarios aggregated rate has dropped to zero, otherwise returns an empty + // flux private Mono publishing() { return transactionGenerator .scenarios() .map(Scenario::getRate) .reduce(0.0, (c, n) -> c + n) .filter(sum -> sum <= 0) - .flatMap(n -> UNKNOWN); + .flatMap(n -> getHealthForZeroRate()); } - // Returns up if any subscription is running and its rate is above zero, otherwise returns unknown + // Returns up if any subscription is running and its rate is above zero, otherwise returns down or unknown private Mono subscribing() { return mirrorSubscriber .getSubscriptions() @@ -65,7 +68,7 @@ private Mono subscribing() { .reduce(0.0, (cur, next) -> cur + next) .filter(sum -> sum > 0) .flatMap(n -> UP) - .switchIfEmpty(UNKNOWN); + .switchIfEmpty(getHealthForZeroRate()); } private Mono restNetworkStakeHealth() { @@ -98,4 +101,8 @@ private Mono restNetworkStakeHealth() { return health(status, statusMessage); }); } + + private Mono getHealthForZeroRate() { + return releaseHealthProperties.isFailWhenInactive() ? DOWN : UNKNOWN; + } } diff --git a/monitor/src/main/java/org/hiero/mirror/monitor/health/ReleaseHealthProperties.java b/monitor/src/main/java/org/hiero/mirror/monitor/health/ReleaseHealthProperties.java index 2915f28972d..90f28f919ea 100644 --- a/monitor/src/main/java/org/hiero/mirror/monitor/health/ReleaseHealthProperties.java +++ b/monitor/src/main/java/org/hiero/mirror/monitor/health/ReleaseHealthProperties.java @@ -19,4 +19,6 @@ public class ReleaseHealthProperties { private Duration cacheExpiry = Duration.ofSeconds(30); private boolean enabled = false; + + private boolean failWhenInactive = false; } diff --git a/monitor/src/test/java/org/hiero/mirror/monitor/health/ClusterHealthIndicatorTest.java b/monitor/src/test/java/org/hiero/mirror/monitor/health/ClusterHealthIndicatorTest.java index 9fb5288a6ae..5fadf0725d5 100644 --- a/monitor/src/test/java/org/hiero/mirror/monitor/health/ClusterHealthIndicatorTest.java +++ b/monitor/src/test/java/org/hiero/mirror/monitor/health/ClusterHealthIndicatorTest.java @@ -49,22 +49,36 @@ class ClusterHealthIndicatorTest { @Mock private TransactionGenerator transactionGenerator; + @Mock + private ReleaseHealthProperties releaseHealthProperties; + @InjectMocks private ClusterHealthIndicator clusterHealthIndicator; @ParameterizedTest @CsvSource({ - "1.0, 1.0, 200, UP", // healthy - "0.0, 1.0, 200, UNKNOWN", // publishing inactive - "1.0, 0.0, 200, UNKNOWN", // subscribing inactive - "0.0, 0.0, 200, UNKNOWN", // publishing and subscribing inactive - "1.0, 1.0, 400, UNKNOWN", // unknown network stake - "1.0, 1.0, 500, DOWN", // network stake down - "0.0, 0.0, 500, DOWN", // publishing and subscribing inactive and network stake down - "0.0, 1.0, 500, DOWN", // network stake down and publishing inactive - "1.0, 0.0, 500, DOWN", // network stake down and subscribing inactive + "1.0, 1.0, 200, UP, false", // healthy + "0.0, 1.0, 200, UNKNOWN, false", // publishing inactive + "1.0, 0.0, 200, UNKNOWN, false", // subscribing inactive + "0.0, 0.0, 200, UNKNOWN, false", // publishing and subscribing inactive + "1.0, 1.0, 400, UNKNOWN, false", // unknown network stake + "1.0, 1.0, 500, DOWN, false", // network stake down + "0.0, 0.0, 500, DOWN, false", // publishing and subscribing inactive and network stake down + "0.0, 1.0, 500, DOWN, false", // network stake down and publishing inactive + "1.0, 0.0, 500, DOWN, false", // network stake down and subscribing inactive + "1.0, 1.0, 200, UP, true", // healthy + "0.0, 1.0, 500, DOWN, true", // publishing inactive + "1.0, 0.0, 500, DOWN, true", // subscribing inactive + "0.0, 0.0, 500, DOWN, true", // publishing and subscribing inactive + "1.0, 1.0, 400, UNKNOWN, true", // unknown network stake + "1.0, 1.0, 500, DOWN, true", // network stake down + "0.0, 0.0, 500, DOWN, true", // publishing and subscribing inactive and network stake down + "0.0, 1.0, 500, DOWN, true", // network stake down and publishing inactive + "1.0, 0.0, 500, DOWN, true", // network stake down and subscribing inactive }) - void health(double publishRate, double subscribeRate, int networkStatusCode, Status status) { + void health( + double publishRate, double subscribeRate, int networkStatusCode, Status status, boolean failWhenInactive) { + when(releaseHealthProperties.isFailWhenInactive()).thenReturn(failWhenInactive); when(transactionGenerator.scenarios()).thenReturn(Flux.just(publishScenario(publishRate))); when(mirrorSubscriber.getSubscriptions()).thenReturn(Flux.just(subscribeScenario(subscribeRate))); when(restApiClient.getNetworkStakeStatusCode())