745 - Tracing

server/apps/server-app/build.gradle.kts

@@ -9,8 +9,6 @@ dependencies {
     implementation("io.awspring.cloud:spring-cloud-aws-starter-s3")
     implementation("io.awspring.cloud:spring-cloud-aws-starter-secrets-manager")
     implementation("io.awspring.cloud:spring-cloud-aws-starter-sqs")
-    implementation("io.micrometer:micrometer-registry-prometheus")
-    implementation(libs.loki.logback.appender)
     implementation(libs.org.springdoc.springdoc.openapi.starter.common)
     implementation("org.springframework.ai:spring-ai-openai-spring-boot-starter:${rootProject.libs.versions.spring.ai.get()}")
     implementation("org.springframework.ai:spring-ai-pgvector-store-spring-boot-starter:${rootProject.libs.versions.spring.ai.get()}")
@@ -55,6 +53,7 @@ dependencies {
     implementation(project(":server:libs:config:liquibase-config"))
     implementation(project(":server:libs:config:logback-config"))
     implementation(project(":server:libs:config:messages-config"))
+    implementation(project(":server:libs:config:observability-config"))
     implementation(project(":server:libs:config:rest-config"))
     implementation(project(":server:libs:config:security-config"))
     implementation(project(":server:libs:config:static-resources-config"))

server/apps/server-app/src/main/resources/config/application-dev.yml

@@ -36,3 +36,12 @@ bytechef:
   security:
     remember-me:
       key: e48612ba1fd46fa7089fe9f5085d8d164b53ffb2
+
+management:
+  tracing:
+    sampling:
+      probability: 1.0
+  zipkin:
+    tracing:
+      endpoint: http://localhost:4318/v1/traces
+      encoding: PROTO3

server/apps/server-app/src/main/resources/config/application.yml

@@ -217,6 +217,8 @@ bytechef:
   loki:
     appender:
       level: "OFF"
+      http:
+        url: http://localhost:3100/loki/api/v1/push
   mail:
     base-url: ${bytechef.public-url}
     from: [email protected]

server/docker/grafana/grafana.ini

@@ -1,3 +1,6 @@
+[feature_toggles]
+enable = tempoSearch tempoBackendSearch tempoApmTable traceToMetrics
+
 [users]
 default_theme = light
 

server/docker/grafana/provisioning/grafana-datasources/datasource.yml

@@ -8,9 +8,43 @@ datasources:
     editable: false
     jsonData:
       httpMethod: POST
-#        exemplarTraceIdDestinations:
-#          - name: trace_id
-#            datasourceUid: tempo
+      exemplarTraceIdDestinations:
+        - name: trace_id
+          datasourceUid: tempo
+  - name: Tempo
+    type: tempo
+    access: proxy
+    orgId: 1
+    url: http://host.docker.internal:3200
+    basicAuth: false
+    isDefault: true
+    version: 1
+    editable: false
+    apiVersion: 1
+    uid: tempo
+    jsonData:
+      httpMethod: GET
+      tracesToLogsV2:
+        datasourceUid: 'loki'
+        spanStartTimeShift: '-1h'
+        spanEndTimeShift: '1h'
+        filterByTraceID: true
+        filterBySpanID: true
+      tracesToMetrics:
+        datasourceUid: Prometheus
+      tracesToLogs:
+        datasourceUid: loki
+        mapTagNamesEnabled: true
+        filterByTraceID: true
+        filterBySpanID: true
+        spanStartTimeShift: '-10m'
+        spanEndTimeShift: '10m'
+      lokiSearch:
+        datasourceUid: loki
+      serviceMap:
+        datasourceUid: Prometheus
+      nodeGraph:
+        enabled: true
   - name: Loki
     type: loki
     uid: loki
@@ -24,8 +58,8 @@ datasources:
     apiVersion: 1
     jsonData:
       maxLines: 50
-#      derivedFields:
-#        - datasourceUid: tempo
-#          matcherRegex: '.+ --- \[.+\] \[.+\] \[(\w*)-\w*\] .+'
-#          name: TraceID
-#          url: $${__value.raw}
+      derivedFields:
+        - datasourceUid: tempo
+          matcherRegex: '.+ --- \[.+\] \[.+\] \[(\w*)-\w*\] .+'
+          name: TraceID
+          url: $${__value.raw}

server/docker/monitoring.yml

@@ -5,8 +5,12 @@ services:
     image: prom/prometheus:v3.0.1
     extra_hosts: [ 'host.docker.internal:host-gateway' ]
     volumes:
+      - prometheus:/prometheus
       - ./prometheus/:/etc/prometheus/
     command:
+      - --enable-feature=exemplar-storage
+      - --enable-feature=otlp-write-receiver
+      - --web.enable-remote-write-receiver
       - --config.file=/etc/prometheus/prometheus-${profile:-dev}.yml
     ports:
       - 9090:9090
@@ -21,6 +25,29 @@ services:
     ports:
       - 3100:3100
 
+  tempo-init: # See: https://github.com/grafana/tempo/issues/1657#issuecomment-2300351643
+    image: &tempoImage grafana/tempo:2.6.1
+    user: root
+    entrypoint:
+      - "chown"
+      - "10001:10001"
+      - "/var/tempo"
+    volumes:
+      - tempo:/var/tempo
+
+  tempo:
+    container_name: tempo
+    image: *tempoImage
+    extra_hosts: [ 'host.docker.internal:host-gateway' ]
+    command: [ '-config.file=/etc/tempo.yml' ]
+    depends_on: [ 'tempo-init' ]
+    volumes:
+      - tempo:/var/tempo
+      - ./tempo/tempo-${profile:-dev}.yml:/etc/tempo.yml
+    ports:
+      - "3200:3200"    # tempo
+      - "4318:4318"  # otlp http
+
   grafana:
     image: grafana/grafana:11.4.0
     extra_hosts: [ 'host.docker.internal:host-gateway' ]
@@ -34,3 +61,8 @@ services:
       - ./grafana/provisioning/grafana-dashboards:/etc/grafana/provisioning/dashboards
     ports:
       - 3000:3000
+volumes:
+  prometheus:
+    driver: local
+  tempo:
+    driver: local

server/docker/tempo/tempo-dev.yml

@@ -0,0 +1,23 @@
+server:
+  http_listen_port: 3200
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        http:
+storage:
+  trace:
+    backend: local
+    local:
+      path: /tmp/tempo/blocks
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+  storage:
+    path: /tmp/tempo/generator/wal
+    remote_write:
+      - url: http://host.docker.internal:9090/api/v1/write
+        send_exemplars: true
+overrides:
+  metrics_generator_processors: [service-graphs, span-metrics]

server/libs/config/app-config/src/main/java/com/bytechef/config/ApplicationProperties.java

@@ -283,8 +283,17 @@ public void setAppender(Appender appender) {
         }
 
         public static class Appender {
+            private Http http;
             private Level level = Level.OFF;
 
+            public Http getHttp() {
+                return http;
+            }
+
+            public void setHttp(Http http) {
+                this.http = http;
+            }
+
             public enum Level {
                 DEBUG, ERROR, FATAL, INFO, OFF, TRACE, WARN
             }
@@ -296,6 +305,18 @@ public Level getLevel() {
             public void setLevel(Level level) {
                 this.level = level;
             }
+
+            public static class Http {
+                private String url;
+
+                public String getUrl() {
+                    return url;
+                }
+
+                public void setUrl(String url) {
+                    this.url = url;
+                }
+            }
         }
     }
 

server/libs/config/logback-config/src/main/resources/logback-spring.xml

@@ -8,33 +8,20 @@
     <!-- The FILE and ASYNC appenders are here as examples for a production configuration -->
     <property name="FILE_LOG_PATTERN" value="${FILE_LOG_PATTERN:-%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd'T'HH:mm:ss.SSSXXX}} ${LOG_LEVEL_PATTERN:-%5p} ${PID:- } &#45;&#45;&#45; [%t] %-40.40logger{39} : %crlf(%m) %n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}"/>
 
-    <springProperty name="applicationName" source="spring.application.name"/>
+    <springProperty name="applicationName" source="spring.application.name" />
     <springProperty name="log.level" source="logging.level.root" defaultValue="INFO" />
     <springProperty name="loki.appender.level" source="bytechef.loki.appender.level" defaultValue="OFF" />
+    <springProperty name="loki.appender.http.url" source="bytechef.loki.appender.http.url" />
 
     <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
     <include resource="org/springframework/boot/logging/logback/console-appender.xml" />
 
-    <!-- The FILE and ASYNC appenders are here as examples for a production configuration -->
-    <!--
-    <include resource="org/springframework/boot/logging/logback/file-appender.xml" />
-
-    <appender name="ASYNC" class="ch.qos.logback.classic.AsyncAppender">
-        <queueSize>512</queueSize>
-        <appender-ref ref="FILE"/>
-    </appender>
-
-    <root level="${logging.level.root}">
-        <appender-ref ref="ASYNC"/>
-    </root>
-    -->
-
     <appender name="LOKI" class="com.github.loki4j.logback.Loki4jAppender">
         <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
             <level>${loki.appender.level}</level>
         </filter>
         <http>
-            <url>http://localhost:3100/loki/api/v1/push</url>
+            <url>${loki.appender.http.url}</url>
         </http>
         <format>
             <label>
@@ -47,6 +34,22 @@
         </format>
     </appender>
 
+    <!-- The FILE and ASYNC appenders are here as examples for a production configuration -->
+    <!--
+    <include resource="org/springframework/boot/logging/logback/file-appender.xml" />
+
+    <appender name="ASYNC" class="ch.qos.logback.classic.AsyncAppender">
+        <queueSize>512</queueSize>
+        <appender-ref ref="FILE"/>
+    </appender>
+
+    <root level="${logging.level.root}">
+        <appender-ref ref="ASYNC"/>
+    </root>
+    -->
+
+    <!--<logger name="com.bytechef" level="INFO"/>-->
+
     <logger name="angus.activation" level="WARN"/>
     <logger name="ch.qos.logback" level="WARN"/>
     <logger name="com.ryantenney" level="WARN"/>

server/libs/config/observability-config/build.gradle.kts

@@ -0,0 +1,7 @@
+dependencies {
+    implementation("io.micrometer:micrometer-registry-prometheus")
+    implementation("io.micrometer:micrometer-tracing-bridge-brave")
+    implementation("io.zipkin.contrib.otel:encoder-brave:0.1.0")
+    implementation(libs.loki.logback.appender)
+    implementation("org.springframework:spring-context")
+}

server/libs/config/observability-config/src/main/java/com/bytechef/observability/config/BraveOtlpConfig.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2023-present ByteChef Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.bytechef.observability.config;
+
+import brave.handler.MutableSpan;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import zipkin2.reporter.BytesEncoder;
+import zipkin2.reporter.otel.brave.OtlpProtoV1Encoder;
+
+/**
+ * @author Matija Petanjek
+ */
+@Configuration(proxyBeanMethods = false)
+public class BraveOtlpConfig {
+
+    @Bean
+    BytesEncoder<MutableSpan> otlpMutableSpanBytesEncoder() {
+        return OtlpProtoV1Encoder.create();
+    }
+
+}

settings.gradle.kts

@@ -86,6 +86,7 @@ include("server:libs:config:jackson-config")
 include("server:libs:config:jdbc-config")
 include("server:libs:config:logback-config")
 include("server:libs:config:messages-config")
+include("server:libs:config:observability-config")
 include("server:libs:config:liquibase-config")
 include("server:libs:config:rest-config")
 include("server:libs:config:static-resources-config")