diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 7a0074923d9..0b2cbf9c848 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -26,6 +26,7 @@ com-squareup-javapoet = "com.squareup:javapoet:1.13.0" commons-validator = "commons-validator:commons-validator:1.9.0" io-swagger-core-v3-swagger-annotations = "io.swagger.core.v3:swagger-annotations:2.2.26" io-swagger-parser-v3-swagger-parser = "io.swagger.parser.v3:swagger-parser:2.1.24" +loki-logback-appender = "com.github.loki4j:loki-logback-appender:1.5.2" org-apache-activemq-artemis-jakarta-server = "org.apache.activemq:artemis-jakarta-server:2.38.0" org-apache-poi-poi-ooxml = "org.apache.poi:poi-ooxml:5.3.0" org-eclipse-jgit-org-eclipse-jgit = "org.eclipse.jgit:org.eclipse.jgit:7.0.0.202409031743-r" diff --git a/server/apps/server-app/build.gradle.kts b/server/apps/server-app/build.gradle.kts index a6d67ac3ae9..ef2e4dc411b 100644 --- a/server/apps/server-app/build.gradle.kts +++ b/server/apps/server-app/build.gradle.kts @@ -10,6 +10,7 @@ dependencies { implementation("io.awspring.cloud:spring-cloud-aws-starter-secrets-manager") implementation("io.awspring.cloud:spring-cloud-aws-starter-sqs") implementation("io.micrometer:micrometer-registry-prometheus") + implementation(libs.loki.logback.appender) implementation(libs.org.springdoc.springdoc.openapi.starter.common) implementation("org.springframework.ai:spring-ai-openai-spring-boot-starter:${rootProject.libs.versions.spring.ai.get()}") implementation("org.springframework.ai:spring-ai-pgvector-store-spring-boot-starter:${rootProject.libs.versions.spring.ai.get()}") diff --git a/server/apps/server-app/src/main/resources/config/application-dev.yml b/server/apps/server-app/src/main/resources/config/application-dev.yml index 326c92ea7eb..8b933dbffd4 100644 --- a/server/apps/server-app/src/main/resources/config/application-dev.yml +++ b/server/apps/server-app/src/main/resources/config/application-dev.yml @@ -2,6 +2,7 @@ logging: level: ROOT: INFO com.bytechef: DEBUG + config: "classpath:logback-spring-dev.xml" server: port: 9555 diff --git a/server/apps/server-app/src/main/resources/config/application-prod.yml b/server/apps/server-app/src/main/resources/config/application-prod.yml index 0f946569f81..f33adc9b6c8 100644 --- a/server/apps/server-app/src/main/resources/config/application-prod.yml +++ b/server/apps/server-app/src/main/resources/config/application-prod.yml @@ -2,6 +2,7 @@ logging: level: ROOT: INFO com.bytechef: INFO + config: "classpath:logback-spring-prod.xml" management: prometheus: diff --git a/server/docker/grafana/grafana.ini b/server/docker/grafana/grafana.ini new file mode 100644 index 00000000000..4b3277e515c --- /dev/null +++ b/server/docker/grafana/grafana.ini @@ -0,0 +1,7 @@ +[users] +default_theme = light + +[smtp] +enabled = true +host = host.docker.internal:25 +skip_verify = true diff --git a/server/docker/grafana/provisioning/dashboards/dashboard.yml b/server/docker/grafana/provisioning/dashboards/dashboard.yml new file mode 100644 index 00000000000..31d21dbf56b --- /dev/null +++ b/server/docker/grafana/provisioning/dashboards/dashboard.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +providers: + - name: dashboards + type: file + disableDeletion: true + editable: true + options: + path: /etc/grafana/provisioning/dashboards + foldersFromFilesStructure: true diff --git a/server/docker/grafana/provisioning/dashboards/logs_traces_metrics.json b/server/docker/grafana/provisioning/dashboards/logs_traces_metrics.json new file mode 100644 index 00000000000..32456f337d0 --- /dev/null +++ b/server/docker/grafana/provisioning/dashboards/logs_traces_metrics.json @@ -0,0 +1,102 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "description": "", + "gridPos": { + "h": 10, + "w": 23, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": true, + "showCommonLabels": true, + "showLabels": true, + "showTime": true, + "sortOrder": "Ascending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "builder", + "expr": "{app=~\".+\"} |= `$traceId`", + "queryType": "range", + "refId": "A" + } + ], + "title": "Logs with trace ID $traceId", + "type": "logs" + } + ], + "schemaVersion": 1, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false + }, + "description": "Trace Id", + "hide": 0, + "includeAll": false, + "label": "Trace ID", + "multi": false, + "name": "traceId", + "options": [], + "query": "", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Logs", + "uid": "edz2d9w1so8aoa", + "version": 1, + "weekStart": "" +} diff --git a/server/docker/loki/loki-dev.yml b/server/docker/loki/loki-dev.yml new file mode 100644 index 00000000000..7fc3a52fc27 --- /dev/null +++ b/server/docker/loki/loki-dev.yml @@ -0,0 +1,41 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2024-12-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + + +limits_config: + max_global_streams_per_user: 0 + max_cache_freshness_per_query: '10m' + # reject_old_samples: true + # reject_old_samples_max_age: 90m + ingestion_rate_mb: 20 + ingestion_burst_size_mb: 40 + # parallelize queries in 15min intervals + # split_queries_by_interval: 15m + # volume_enabled: true + +#ruler: +# alertmanager_url: http://localhost:9093 diff --git a/server/docker/monitoring.yml b/server/docker/monitoring.yml index b303e969754..9d0d2df0555 100644 --- a/server/docker/monitoring.yml +++ b/server/docker/monitoring.yml @@ -10,3 +10,27 @@ services: - --config.file=/etc/prometheus/prometheus-${profile:-dev}.yml ports: - 9090:9090 + + loki: + image: grafana/loki:3.2.2 + extra_hosts: [ 'host.docker.internal:host-gateway' ] + volumes: + - ./loki/:/etc/loki/ + command: + - --config.file=/etc/loki/loki-${profile:-dev}.yml + ports: + - 3100:3100 + + grafana: + image: grafana/grafana:11.4.0 + extra_hosts: [ 'host.docker.internal:host-gateway' ] + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_AUTH_DISABLE_LOGIN_FORM=true + volumes: + - ./grafana/grafana.ini:/etc/grafana/grafana.ini + - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources + - ./grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards + ports: + - 3000:3000 diff --git a/server/libs/config/logback-config/src/main/resources/logback-spring-dev.xml b/server/libs/config/logback-config/src/main/resources/logback-spring-dev.xml new file mode 100644 index 00000000000..884aff8c452 --- /dev/null +++ b/server/libs/config/logback-config/src/main/resources/logback-spring-dev.xml @@ -0,0 +1,28 @@ + + + + + + + + + + + http://localhost:3100/loki/api/v1/push + + + + + ${FILE_LOG_PATTERN} + + true + + + + + + + + diff --git a/server/libs/config/logback-config/src/main/resources/logback-spring-prod.xml b/server/libs/config/logback-config/src/main/resources/logback-spring-prod.xml new file mode 100644 index 00000000000..8a23b128bd7 --- /dev/null +++ b/server/libs/config/logback-config/src/main/resources/logback-spring-prod.xml @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/server/libs/config/logback-config/src/main/resources/logback-spring.xml b/server/libs/config/logback-config/src/main/resources/logback-spring.xml index e43e1722141..5f4c2596938 100644 --- a/server/libs/config/logback-config/src/main/resources/logback-spring.xml +++ b/server/libs/config/logback-config/src/main/resources/logback-spring.xml @@ -6,9 +6,9 @@ - + + @@ -61,12 +61,6 @@ - - - - - - true