Skip to content

Commit d164132

Browse files
ish-hccish-hcc
authored
Merge pull request #211 from ish-hcc/main
Update Gradle & Add readyz API
2 parents e4b5bdd + 6029340 commit d164132

File tree

10 files changed

+181
-57
lines changed

10 files changed

+181
-57
lines changed

‎java-module/build.gradle‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@ plugins {
22
id 'java'
33
id 'org.springframework.boot' version '3.2.12'
44
id 'io.spring.dependency-management' version '1.1.7'
5-
id 'org.asciidoctor.jvm.convert' version '3.3.2'
6-
id 'com.epages.restdocs-api-spec' version '0.18.2'
5+
id 'org.asciidoctor.jvm.convert' version '4.0.5'
6+
id 'com.epages.restdocs-api-spec' version '0.19.4'
77
id 'com.diffplug.spotless' version '7.1.0'
88
}
99

@@ -12,8 +12,12 @@ bootJar.enabled = false
1212
allprojects {
1313
group = 'mcmp.mc.observability'
1414
version = '0.3.1'
15-
sourceCompatibility = JavaVersion.VERSION_17
16-
targetCompatibility = JavaVersion.VERSION_17
15+
16+
java {
17+
toolchain {
18+
languageVersion = JavaLanguageVersion.of(17)
19+
}
20+
}
1721

1822
repositories {
1923
mavenCentral()

‎java-module/docker-compose.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ services:
2424
condition: service_healthy
2525
mc-observability-grafana:
2626
condition: service_healthy
27+
mc-observability-rabbitmq:
28+
condition: service_healthy
2729
environment:
2830
- JAVA_TOOL_OPTIONS=-javaagent:/opentelemetry-javaagent.jar -XX:-UseContainerSupport
2931
- TUMBLEBUG_URL=${TUMBLEBUG_URL:-http://mc-infra-manager:1323}
@@ -704,6 +706,12 @@ services:
704706
networks:
705707
- internal_network
706708
- external_network
709+
healthcheck:
710+
start_period: 120s
711+
test: rabbitmq-diagnostics -q ping
712+
interval: 10s
713+
timeout: 10s
714+
retries: 30
707715

708716
mc-observability-rabbitmq-init-volumes:
709717
image: busybox:stable

‎java-module/gradle.properties‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
org.gradle.configuration-cache=true

‎java-module/gradle/wrapper/gradle-wrapper.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
distributionBase=GRADLE_USER_HOME
22
distributionPath=wrapper/dists
3-
distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip
3+
distributionUrl=https\://services.gradle.org/distributions/gradle-9.1.0-bin.zip
44
networkTimeout=10000
55
validateDistributionUrl=true
66
zipStoreBase=GRADLE_USER_HOME

‎java-module/grafana/run.sh‎

Lines changed: 0 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -180,52 +180,6 @@ RESPONSE=$(curl -s -b ~/grafana-cookie -XGET \
180180
DATA_SOURCE_INFLUXDB_UID=$(echo $RESPONSE | sed -n 's/.*"uid":"\([^"]*\)".*/\1/p')
181181
echo DATA_SOURCE_INFLUXDB_UID=$DATA_SOURCE_INFLUXDB_UID >> ~/env.grafana
182182

183-
echo "[*] Checking existing rules..."
184-
RESPONSE=$(curl -s -b ~/grafana-cookie -XGET \
185-
"http://127.0.0.1:3000/api/ruler/grafana/api/v1/rules?subtype=cortex" \
186-
-H 'Accept: application/json, text/plain, */*' \
187-
-H 'Content-Type: application/json' \
188-
-H 'Origin: http://127.0.0.1:3000')
189-
190-
LOG_RULE_EXISTS=$(echo $RESPONSE | grep -q '"log"' && echo "true" || echo "false")
191-
METRIC_RULE_EXISTS=$(echo $RESPONSE | grep -q '"metric"' && echo "true" || echo "false")
192-
193-
if [ "$LOG_RULE_EXISTS" = "true" ]; then
194-
echo "[*] Log rule already exists, skipping initialization."
195-
else
196-
echo "[*] Initializing log rule..."
197-
HTTP_STATUS=$(curl -s -w "\n%{http_code}" -b ~/grafana-cookie -XPOST \
198-
"http://127.0.0.1:3000/api/ruler/grafana/api/v1/rules/$FOLDER_UID?subtype=cortex" \
199-
-H 'Accept: */*' \
200-
-H 'Content-Type: application/json' \
201-
-H 'Origin: http://127.0.0.1:3000' \
202-
-d '{"name":"log","rules":[{"grafana_alert":{"title":"log-rule-init","condition":"C","data":[{"refId":"A","datasourceUid":"'$DATA_SOURCE_LOKI_UID'","queryType":"range","relativeTimeRange":{"from":600,"to":0},"model":{"refId":"A","instant":true,"expr":"{job=\"non-existent-job\", uuid=\"definitely-not-real\"} |= \"never-happens-keyword\"","queryType":"range","editorMode":"code","direction":"backward"}},{"datasourceUid":"__expr__","model":{"refId":"B","datasource":{"type":"__expr__","uid":"__expr__","name":"Expression"},"type":"reduce","reducer":"last","conditions":[{"type":"query","reducer":{"params":[],"type":"avg"},"operator":{"type":"and"},"query":{"params":[]},"evaluator":{"params":[0,0],"type":"gt"}}],"expression":"A"},"refId":"B","queryType":"expression"},{"refId":"C","datasourceUid":"__expr__","queryType":"","model":{"refId":"C","type":"threshold","datasource":{"uid":"__expr__","type":"__expr__"},"conditions":[{"type":"query","evaluator":{"params":[0],"type":"gt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"}}],"expression":"B"}}],"is_paused":true,"no_data_state":"NoData","exec_err_state":"Error","notification_settings":{"receiver":"o11y"}},"annotations":{},"labels":{},"for":"0s"}],"interval":"10m"}' | tail -n1)
203-
if [[ $HTTP_STATUS =~ ^2[0-9][0-9]$ ]]; then
204-
echo "[*] Successfully initialized log rule!"
205-
else
206-
echo "[!] Failed to initialize log rule."
207-
exit 1
208-
fi
209-
fi
210-
211-
if [ "$METRIC_RULE_EXISTS" = "true" ]; then
212-
echo "[*] Metric rule already exists, skipping initialization."
213-
else
214-
echo "[*] Initializing metric rule..."
215-
HTTP_STATUS=$(curl -s -w "\n%{http_code}" -b ~/grafana-cookie -XPOST \
216-
"http://127.0.0.1:3000/api/ruler/grafana/api/v1/rules/$FOLDER_UID?subtype=cortex" \
217-
-H 'Accept: */*' \
218-
-H 'Content-Type: application/json' \
219-
-H 'Origin: http://127.0.0.1:3000' \
220-
-d '{"name":"metric","rules":[{"grafana_alert":{"title":"metric-rule-init","condition":"C","data":[{"refId":"A","relativeTimeRange":{"from":600,"to":0},"queryType":"","datasourceUid":"'$DATA_SOURCE_INFLUXDB_UID'","model":{"refId":"A","hide":false,"datasource":{"uid":"'$DATA_SOURCE_INFLUXDB_UID'","type":"influxdb"},"instant":true,"query":"SELECT 100-mean(\"usage_idle\") FROM \"cpu\" GROUP BY \"host\"","rawQuery":true,"resultFormat":"time_series"}},{"refId":"C","datasourceUid":"__expr__","queryType":"","model":{"refId":"C","type":"threshold","datasource":{"uid":"__expr__","type":"__expr__"},"conditions":[{"type":"query","evaluator":{"params":[0],"type":"gt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"}}],"expression":"A"}}],"is_paused":true,"no_data_state":"NoData","exec_err_state":"Error","notification_settings":{"receiver":"o11y"}},"annotations":{},"labels":{},"for":"0s"}],"interval":"1m"}' | tail -n1)
221-
if [[ $HTTP_STATUS =~ ^2[0-9][0-9]$ ]]; then
222-
echo "[*] Successfully initialized metric rule!"
223-
else
224-
echo "[!] Failed to initialize metric rule."
225-
exit 1
226-
fi
227-
fi
228-
229183
echo "[*] Deleting cookie..."
230184
rm -f ~/grafana-cookie
231185

‎java-module/mc-o11y-manager/build.gradle‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ dependencies {
2626
implementation 'org.apache.sshd:sshd-core:2.15.0'
2727
implementation 'org.apache.sshd:sshd-sftp:2.15.0'
2828
implementation 'org.springframework.retry:spring-retry'
29+
implementation 'org.apache.httpcomponents.client5:httpclient5:5.2.3'
2930
compileOnly 'org.projectlombok:lombok'
3031
developmentOnly 'org.springframework.boot:spring-boot-devtools'
3132
runtimeOnly 'org.mariadb.jdbc:mariadb-java-client'
@@ -34,7 +35,10 @@ dependencies {
3435
testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
3536
implementation 'org.influxdb:influxdb-java'
3637

37-
implementation 'org.springframework.boot:spring-boot-starter-amqp'
38+
implementation ('org.springframework.boot:spring-boot-starter-amqp') {
39+
exclude group: 'org.springframework.amqp', module: 'spring-rabbit'
40+
}
41+
implementation 'org.springframework.amqp:spring-rabbit:3.2.1'
3842
implementation 'org.springframework.boot:spring-boot-starter-webflux'
3943
implementation 'org.springframework.boot:spring-boot-starter-mail'
4044
implementation 'org.springframework.boot:spring-boot-starter-thymeleaf'
@@ -87,7 +91,7 @@ jar {
8791
}
8892

8993
bootJar {
90-
archivesBaseName = "mc-o11y-manager"
94+
archiveBaseName = "mc-o11y-manager"
9195
archiveFileName = "mc-o11y-manager.jar"
9296
archiveVersion = "0.3.1"
9397
}

‎java-module/mc-o11y-manager/src/main/java/com/mcmp/o11ymanager/manager/controller/HealthController.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package com.mcmp.o11ymanager.manager.controller;
2+
3+
import com.mcmp.o11ymanager.manager.service.HealthServiceImpl;
4+
import java.util.Map;
5+
import lombok.RequiredArgsConstructor;
6+
import lombok.extern.slf4j.Slf4j;
7+
import org.springframework.http.ResponseEntity;
8+
import org.springframework.web.bind.annotation.GetMapping;
9+
import org.springframework.web.bind.annotation.RequestMapping;
10+
import org.springframework.web.bind.annotation.RestController;
11+
12+
@Slf4j
13+
@RestController
14+
@RequiredArgsConstructor
15+
@RequestMapping("/api/o11y")
16+
public class HealthController {
17+
private final HealthServiceImpl healthService;
18+
19+
@GetMapping("/readyz")
20+
public ResponseEntity<Map<String, Object>> readyz() {
21+
Map<String, Object> healthStatus = healthService.checkApiHealth();
22+
String status = (String) healthStatus.get("status");
23+
24+
if ("DOWN".equals(status)) {
25+
return ResponseEntity.status(503).body(healthStatus);
26+
} else {
27+
return ResponseEntity.ok(healthStatus);
28+
}
29+
}
30+
}

‎java-module/mc-o11y-manager/src/main/java/com/mcmp/o11ymanager/manager/service/HealthServiceImpl.java‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
package com.mcmp.o11ymanager.manager.service;
2+
3+
import java.util.Base64;
4+
import java.util.HashMap;
5+
import java.util.Map;
6+
import lombok.RequiredArgsConstructor;
7+
import lombok.extern.slf4j.Slf4j;
8+
import org.springframework.beans.factory.annotation.Value;
9+
import org.springframework.http.HttpEntity;
10+
import org.springframework.http.HttpHeaders;
11+
import org.springframework.http.HttpMethod;
12+
import org.springframework.http.ResponseEntity;
13+
import org.springframework.http.client.SimpleClientHttpRequestFactory;
14+
import org.springframework.stereotype.Service;
15+
import org.springframework.web.client.RestTemplate;
16+
17+
@Service
18+
@RequiredArgsConstructor
19+
@Slf4j
20+
public class HealthServiceImpl {
21+
22+
@Value("${feign.cb-tumblebug.url}")
23+
private String tumblebugURL;
24+
25+
@Value("${feign.cb-tumblebug.id}")
26+
private String tumblebugID;
27+
28+
@Value("${feign.cb-tumblebug.pw}")
29+
private String tumblebugPW;
30+
31+
@Value("${feign.insight.url}")
32+
private String insightURL;
33+
34+
public Map<String, Object> checkExternalServiceHealth(String externalServiceUrl) {
35+
return checkExternalServiceHealth(externalServiceUrl, null, null);
36+
}
37+
38+
public Map<String, Object> checkExternalServiceHealth(
39+
String externalServiceUrl, String username, String password) {
40+
Map<String, Object> externalServiceStatus = new HashMap<>();
41+
ResponseEntity<String> response;
42+
boolean isHealthy;
43+
44+
SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory();
45+
factory.setConnectTimeout(5000);
46+
factory.setReadTimeout(5000);
47+
48+
RestTemplate restTemplate = new RestTemplate(factory);
49+
50+
try {
51+
if (username != null
52+
&& !username.isEmpty()
53+
&& password != null
54+
&& !password.isEmpty()) {
55+
HttpHeaders headers = new HttpHeaders();
56+
String auth = username + ":" + password;
57+
String encodedAuth = Base64.getEncoder().encodeToString(auth.getBytes());
58+
headers.set(HttpHeaders.AUTHORIZATION, "Basic " + encodedAuth);
59+
60+
HttpEntity<String> entity = new HttpEntity<>(headers);
61+
response =
62+
restTemplate.exchange(
63+
externalServiceUrl, HttpMethod.GET, entity, String.class);
64+
} else {
65+
response =
66+
restTemplate.exchange(
67+
externalServiceUrl, HttpMethod.GET, null, String.class);
68+
}
69+
70+
isHealthy = (response.getStatusCode().value() == 200);
71+
} catch (Exception e) {
72+
isHealthy = false;
73+
log.error("Error checking health for external service: " + externalServiceUrl, e);
74+
}
75+
76+
externalServiceStatus.put("status", isHealthy ? "UP" : "DOWN");
77+
78+
return externalServiceStatus;
79+
}
80+
81+
public Map<String, Object> checkApiHealth() {
82+
Map<String, Object> apiHealthStatus = new HashMap<>();
83+
84+
Map<String, Object> tumblebugStatus =
85+
checkExternalServiceHealth(
86+
tumblebugURL + "/tumblebug/readyz", tumblebugID, tumblebugPW);
87+
Map<String, Object> insightStatus =
88+
checkExternalServiceHealth(insightURL + "/api/o11y/insight/predictions/options");
89+
90+
boolean isTumblebugHealthy = "UP".equals(tumblebugStatus.get("status"));
91+
boolean isInsightHealthy = "UP".equals(insightStatus.get("status"));
92+
93+
boolean isApiHealthy = isTumblebugHealthy && isInsightHealthy;
94+
95+
if (isApiHealthy) {
96+
apiHealthStatus.put("message", "All systems are operational");
97+
} else {
98+
StringBuilder message = new StringBuilder("One or more systems are down: ");
99+
if (!isTumblebugHealthy) {
100+
message.append("Tumblebug is down. ");
101+
}
102+
if (!isInsightHealthy) {
103+
message.append("Insight is down. ");
104+
}
105+
apiHealthStatus.put("message", message.toString().trim());
106+
}
107+
108+
return apiHealthStatus;
109+
}
110+
}

‎java-module/mc-o11y-manager/src/main/java/com/mcmp/o11ymanager/manager/service/interfaces/HealthService.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package com.mcmp.o11ymanager.manager.service.interfaces;
2+
3+
import java.util.Map;
4+
5+
public interface HealthService {
6+
7+
public Map<String, Object> checkExternalServiceHealth(String externalServiceUrl);
8+
9+
public Map<String, Object> checkExternalServiceHealth(
10+
String externalServiceUrl, String username, String password);
11+
12+
public Map<String, Object> checkApiHealth();
13+
}

‎java-module/mc-o11y-manager/src/main/java/com/mcmp/o11ymanager/trigger/infrastructure/external/message/config/RabbitMQConfig.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -185,11 +185,11 @@ public SimpleRabbitListenerContainerFactory rabbitListenerContainerFactory(
185185
*/
186186
@Bean
187187
public RabbitListenerErrorHandler jsonParseErrorHandler() {
188-
return (rawMessage, convertedMessage, e) -> {
189-
if (e.getCause() instanceof MessageConversionException) {
190-
log.error("Failed to convert message.", e);
188+
return (amqpMessage, channel, message, exception) -> {
189+
if (exception.getCause() instanceof MessageConversionException) {
190+
log.error("Failed to convert message.", exception);
191191
}
192-
throw e;
192+
throw exception;
193193
};
194194
}
195195
}

0 commit comments

Comments
 (0)