diff --git a/server/apps/server-app/build.gradle.kts b/server/apps/server-app/build.gradle.kts index 7a287cb3232..7664b666966 100644 --- a/server/apps/server-app/build.gradle.kts +++ b/server/apps/server-app/build.gradle.kts @@ -9,8 +9,6 @@ dependencies { implementation("io.awspring.cloud:spring-cloud-aws-starter-s3") implementation("io.awspring.cloud:spring-cloud-aws-starter-secrets-manager") implementation("io.awspring.cloud:spring-cloud-aws-starter-sqs") - implementation("io.micrometer:micrometer-registry-prometheus") - implementation(libs.loki.logback.appender) implementation(libs.org.springdoc.springdoc.openapi.starter.common) implementation("org.springframework.ai:spring-ai-openai-spring-boot-starter:${rootProject.libs.versions.spring.ai.get()}") implementation("org.springframework.ai:spring-ai-pgvector-store-spring-boot-starter:${rootProject.libs.versions.spring.ai.get()}") @@ -55,6 +53,7 @@ dependencies { implementation(project(":server:libs:config:liquibase-config")) implementation(project(":server:libs:config:logback-config")) implementation(project(":server:libs:config:messages-config")) + implementation(project(":server:libs:config:observability-config")) implementation(project(":server:libs:config:rest-config")) implementation(project(":server:libs:config:security-config")) implementation(project(":server:libs:config:static-resources-config")) diff --git a/server/apps/server-app/src/main/resources/config/application-dev.yml b/server/apps/server-app/src/main/resources/config/application-dev.yml index 326c92ea7eb..252ce4ce3d1 100644 --- a/server/apps/server-app/src/main/resources/config/application-dev.yml +++ b/server/apps/server-app/src/main/resources/config/application-dev.yml @@ -36,3 +36,12 @@ bytechef: security: remember-me: key: e48612ba1fd46fa7089fe9f5085d8d164b53ffb2 + +management: + tracing: + sampling: + probability: 1.0 + zipkin: + tracing: + endpoint: http://localhost:4318/v1/traces + encoding: PROTO3 diff --git a/server/apps/server-app/src/main/resources/config/application.yml b/server/apps/server-app/src/main/resources/config/application.yml index e081a12e572..fd6b81ab89a 100644 --- a/server/apps/server-app/src/main/resources/config/application.yml +++ b/server/apps/server-app/src/main/resources/config/application.yml @@ -217,6 +217,8 @@ bytechef: loki: appender: level: "OFF" + http: + url: http://localhost:3100/loki/api/v1/push mail: base-url: ${bytechef.public-url} from: noreply@bytechef.io diff --git a/server/docker/grafana/grafana.ini b/server/docker/grafana/grafana.ini index 4b3277e515c..d26318c5a43 100644 --- a/server/docker/grafana/grafana.ini +++ b/server/docker/grafana/grafana.ini @@ -1,3 +1,6 @@ +[feature_toggles] +enable = tempoSearch tempoBackendSearch tempoApmTable traceToMetrics + [users] default_theme = light diff --git a/server/docker/grafana/provisioning/grafana-datasources/datasource.yml b/server/docker/grafana/provisioning/grafana-datasources/datasource.yml index 4c6e9358c88..713a2da578a 100644 --- a/server/docker/grafana/provisioning/grafana-datasources/datasource.yml +++ b/server/docker/grafana/provisioning/grafana-datasources/datasource.yml @@ -8,9 +8,43 @@ datasources: editable: false jsonData: httpMethod: POST -# exemplarTraceIdDestinations: -# - name: trace_id -# datasourceUid: tempo + exemplarTraceIdDestinations: + - name: trace_id + datasourceUid: tempo + - name: Tempo + type: tempo + access: proxy + orgId: 1 + url: http://host.docker.internal:3200 + basicAuth: false + isDefault: true + version: 1 + editable: false + apiVersion: 1 + uid: tempo + jsonData: + httpMethod: GET + tracesToLogsV2: + datasourceUid: 'loki' + spanStartTimeShift: '-1h' + spanEndTimeShift: '1h' + filterByTraceID: true + filterBySpanID: true + tracesToMetrics: + datasourceUid: Prometheus + tracesToLogs: + datasourceUid: loki + mapTagNamesEnabled: true + filterByTraceID: true + filterBySpanID: true + spanStartTimeShift: '-10m' + spanEndTimeShift: '10m' + lokiSearch: + datasourceUid: loki + serviceMap: + datasourceUid: Prometheus + nodeGraph: + enabled: true - name: Loki type: loki uid: loki @@ -24,8 +58,8 @@ datasources: apiVersion: 1 jsonData: maxLines: 50 -# derivedFields: -# - datasourceUid: tempo -# matcherRegex: '.+ --- \[.+\] \[.+\] \[(\w*)-\w*\] .+' -# name: TraceID -# url: $${__value.raw} + derivedFields: + - datasourceUid: tempo + matcherRegex: '.+ --- \[.+\] \[.+\] \[(\w*)-\w*\] .+' + name: TraceID + url: $${__value.raw} diff --git a/server/docker/monitoring.yml b/server/docker/monitoring.yml index e22cb50c558..72b2e82d8f9 100644 --- a/server/docker/monitoring.yml +++ b/server/docker/monitoring.yml @@ -5,8 +5,12 @@ services: image: prom/prometheus:v3.0.1 extra_hosts: [ 'host.docker.internal:host-gateway' ] volumes: + - prometheus:/prometheus - ./prometheus/:/etc/prometheus/ command: + - --enable-feature=exemplar-storage + - --enable-feature=otlp-write-receiver + - --web.enable-remote-write-receiver - --config.file=/etc/prometheus/prometheus-${profile:-dev}.yml ports: - 9090:9090 @@ -21,6 +25,29 @@ services: ports: - 3100:3100 + tempo-init: # See: https://github.com/grafana/tempo/issues/1657#issuecomment-2300351643 + image: &tempoImage grafana/tempo:2.6.1 + user: root + entrypoint: + - "chown" + - "10001:10001" + - "/var/tempo" + volumes: + - tempo:/var/tempo + + tempo: + container_name: tempo + image: *tempoImage + extra_hosts: [ 'host.docker.internal:host-gateway' ] + command: [ '-config.file=/etc/tempo.yml' ] + depends_on: [ 'tempo-init' ] + volumes: + - tempo:/var/tempo + - ./tempo/tempo-${profile:-dev}.yml:/etc/tempo.yml + ports: + - "3200:3200" # tempo + - "4318:4318" # otlp http + grafana: image: grafana/grafana:11.4.0 extra_hosts: [ 'host.docker.internal:host-gateway' ] @@ -34,3 +61,8 @@ services: - ./grafana/provisioning/grafana-dashboards:/etc/grafana/provisioning/dashboards ports: - 3000:3000 +volumes: + prometheus: + driver: local + tempo: + driver: local diff --git a/server/docker/tempo/tempo-dev.yml b/server/docker/tempo/tempo-dev.yml new file mode 100644 index 00000000000..c32637339fd --- /dev/null +++ b/server/docker/tempo/tempo-dev.yml @@ -0,0 +1,23 @@ +server: + http_listen_port: 3200 +distributor: + receivers: + otlp: + protocols: + http: +storage: + trace: + backend: local + local: + path: /tmp/tempo/blocks +metrics_generator: + registry: + external_labels: + source: tempo + storage: + path: /tmp/tempo/generator/wal + remote_write: + - url: http://host.docker.internal:9090/api/v1/write + send_exemplars: true +overrides: + metrics_generator_processors: [service-graphs, span-metrics] diff --git a/server/libs/config/app-config/src/main/java/com/bytechef/config/ApplicationProperties.java b/server/libs/config/app-config/src/main/java/com/bytechef/config/ApplicationProperties.java index 917cfacc3fe..4e269627c47 100644 --- a/server/libs/config/app-config/src/main/java/com/bytechef/config/ApplicationProperties.java +++ b/server/libs/config/app-config/src/main/java/com/bytechef/config/ApplicationProperties.java @@ -283,8 +283,17 @@ public void setAppender(Appender appender) { } public static class Appender { + private Http http; private Level level = Level.OFF; + public Http getHttp() { + return http; + } + + public void setHttp(Http http) { + this.http = http; + } + public enum Level { DEBUG, ERROR, FATAL, INFO, OFF, TRACE, WARN } @@ -296,6 +305,18 @@ public Level getLevel() { public void setLevel(Level level) { this.level = level; } + + public static class Http { + private String url; + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + } } } diff --git a/server/libs/config/logback-config/src/main/resources/logback-spring.xml b/server/libs/config/logback-config/src/main/resources/logback-spring.xml index e99e52dec4f..163a027071f 100644 --- a/server/libs/config/logback-config/src/main/resources/logback-spring.xml +++ b/server/libs/config/logback-config/src/main/resources/logback-spring.xml @@ -8,33 +8,20 @@ - + + - - - ${loki.appender.level} - http://localhost:3100/loki/api/v1/push + ${loki.appender.http.url} + + + + + diff --git a/server/libs/config/observability-config/build.gradle.kts b/server/libs/config/observability-config/build.gradle.kts new file mode 100644 index 00000000000..8b0b8d84408 --- /dev/null +++ b/server/libs/config/observability-config/build.gradle.kts @@ -0,0 +1,7 @@ +dependencies { + implementation("io.micrometer:micrometer-registry-prometheus") + implementation("io.micrometer:micrometer-tracing-bridge-brave") + implementation("io.zipkin.contrib.otel:encoder-brave:0.1.0") + implementation(libs.loki.logback.appender) + implementation("org.springframework:spring-context") +} diff --git a/server/libs/config/observability-config/src/main/java/com/bytechef/observability/config/BraveOtlpConfig.java b/server/libs/config/observability-config/src/main/java/com/bytechef/observability/config/BraveOtlpConfig.java new file mode 100644 index 00000000000..4e78a4a1daf --- /dev/null +++ b/server/libs/config/observability-config/src/main/java/com/bytechef/observability/config/BraveOtlpConfig.java @@ -0,0 +1,36 @@ +/* + * Copyright 2023-present ByteChef Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.bytechef.observability.config; + +import brave.handler.MutableSpan; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import zipkin2.reporter.BytesEncoder; +import zipkin2.reporter.otel.brave.OtlpProtoV1Encoder; + +/** + * @author Matija Petanjek + */ +@Configuration(proxyBeanMethods = false) +public class BraveOtlpConfig { + + @Bean + BytesEncoder otlpMutableSpanBytesEncoder() { + return OtlpProtoV1Encoder.create(); + } + +} diff --git a/settings.gradle.kts b/settings.gradle.kts index f0196e69a64..99cb45befcf 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -86,6 +86,7 @@ include("server:libs:config:jackson-config") include("server:libs:config:jdbc-config") include("server:libs:config:logback-config") include("server:libs:config:messages-config") +include("server:libs:config:observability-config") include("server:libs:config:liquibase-config") include("server:libs:config:rest-config") include("server:libs:config:static-resources-config")