Adding OpenTelemetry observability features

build.gradle

@@ -1,7 +1,8 @@
 plugins {
     // Spring boot & dependency management:
     // https://docs.spring.io/spring-boot/docs/current/gradle-plugin/reference/htmlsingle/
-    id 'org.springframework.boot' version '3.5.0'
+
+    id 'org.springframework.boot' version '3.5.4'
     // https://docs.spring.io/dependency-management-plugin/docs/current-SNAPSHOT/reference/html/
     id "io.spring.dependency-management" version "1.1.7"
     // Lombok generates getter and setter and more. https://projectlombok.org/
@@ -49,6 +50,13 @@ repositories {
 
 ext {
     springDocVersion = '2.8.9'
+    openTelemetryInstrumentationVersion = "2.18.1"
+}
+
+dependencyManagement {
+    imports {
+        mavenBom("io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom:${openTelemetryInstrumentationVersion}")
+    }
 }
 
 dependencies {
@@ -81,15 +89,24 @@ dependencies {
     implementation "org.springframework.data:spring-data-elasticsearch"
 
     // More flexibility when (de-)serializing json:
-    implementation(group: 'com.networknt', name: 'json-schema-validator', version: '1.5.7');
-    
+    implementation(group: 'com.networknt', name: 'json-schema-validator', version: '1.5.7')
+
     implementation('org.apache.httpcomponents:httpclient:4.5.14')
     implementation('org.apache.httpcomponents:httpclient-cache:4.5.14')
 
     implementation("net.handle:handle-client:9.3.2")
 
+    /* Observability using OpenTelemetry */
+    implementation(platform("io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom:${openTelemetryInstrumentationVersion}"))
+    implementation "io.opentelemetry.instrumentation:opentelemetry-spring-boot-starter"
+    implementation "io.opentelemetry.instrumentation:opentelemetry-instrumentation-annotations"
+    implementation "io.opentelemetry.contrib:opentelemetry-samplers:1.48.0-alpha"
+    implementation "org.springframework.boot:spring-boot-starter-aop"
+    implementation "io.micrometer:micrometer-tracing-bridge-otel"
+    implementation "io.opentelemetry:opentelemetry-exporter-otlp"
+
     testImplementation(platform('org.junit:junit-bom:5.13.1'))
-	testImplementation('org.junit.jupiter:junit-jupiter')
+    testImplementation('org.junit.jupiter:junit-jupiter')
     testImplementation('org.junit.jupiter:junit-jupiter-params')
 
     testImplementation("org.springframework:spring-test")
@@ -187,15 +204,15 @@ jacocoTestReport {
     afterEvaluate {
         //exclude some classes/package from code coverage report
         classDirectories.setFrom(files(classDirectories.files.collect {
-                    fileTree(dir: it, exclude: [\
-            'edu/kit/datamanager/pit/configuration/**', \
-            'edu/kit/datamanager/pit/web/converter/**', \
-            'edu/kit/datamanager/pit/web/ExtendedErrorAttributes**', \
-            'edu/kit/datamanager/pit/web/UncontrolledExceptionHandler**', \
-            'edu/kit/datamanager/pit/common/**', \
-            'edu/kit/datamanager/pit/Application*'
-                        ])
-                }))
+            fileTree(dir: it, exclude: [\
+            'edu/kit/datamanager/pit/configuration/**',                \
+                           'edu/kit/datamanager/pit/web/converter/**',                \
+                           'edu/kit/datamanager/pit/web/ExtendedErrorAttributes**',                \
+                           'edu/kit/datamanager/pit/web/UncontrolledExceptionHandler**',                \
+                           'edu/kit/datamanager/pit/common/**',                \
+                           'edu/kit/datamanager/pit/Application*'
+            ])
+        }))
     }
 }
 

config/application-default.properties

@@ -12,6 +12,8 @@
 
 ### General Spring Boot Settings ###
 # When to include the "message" attribute in HTTP responses on uncatched exceptions.
+spring.application.name=typed-pid-maker
+#spring.profiles.active=default
 server.error.include-message=always
 springdoc.api-docs.enabled=true
 springdoc.swagger-ui.enabled=true
@@ -53,14 +55,13 @@ management.endpoints.web.exposure.include=health, info
 ###############
 ### Logging ###
 ###############
-
 # Logging Settings. Most logging of KIT DM is performed on TRACE level. However, if you
 # plan to enable logging with this granularity it is recommended to this only for 
 # a selection of a few packages. Otherwise, the amount of logging information might be
 # overwhelming.
 #logging.level.root: ERROR
 #logging.level.edu.kit.datamanager.doip:TRACE
-logging.level.edu.kit=WARN
+logging.level.edu.kit=INFO
 #logging.level.org.springframework.transaction: TRACE
 logging.level.org.springframework=WARN
 logging.level.org.springframework.amqp=WARN
@@ -92,7 +93,6 @@ repo.auth.jwtSecret=vkfvoswsohwrxgjaxipuiyyjgubggzdaqrcuupbugxtnalhiegkppdgjgwxs
 ###############################
 ### Keycloak Authentication ###
 ###############################
-
 spring.autoconfigure.exclude=org.keycloak.adapters.springboot.KeycloakAutoConfiguration,org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration
 #keycloakjwt.jwk-url=http://localhost:8080/auth/realms/myrealm/protocol/openid-connect/certs
 #keycloakjwt.resource=keycloak-angular
@@ -109,7 +109,6 @@ spring.autoconfigure.exclude=org.keycloak.adapters.springboot.KeycloakAutoConfig
 ### Elastic Indexing and search endpoint ###
 ######## (requires Elasticsearch 8) ########
 ############################################
-
 # enables search endpoint at /api/v1/search
 repo.search.enabled=false
 repo.search.index=*
@@ -123,7 +122,6 @@ repo.search.url=http://localhost:9200
 #spring.elasticsearch.username=user
 #spring.elasticsearch.password=secret
 #spring.elasticsearch.socket-timeout=10s
-
 # Due to bug in spring cloud gateway
 # https://github.com/spring-cloud/spring-cloud-gateway/issues/3154
 spring.cloud.gateway.proxy.sensitive=content-length
@@ -132,7 +130,6 @@ spring.cloud.gateway.proxy.sensitive=content-length
 #################
 ### Messaging ###
 #################
-
 # Enable (default)/disable messaging. The messaging functionality requires a RabbitMQ
 # server receiving and distributing the messages sent by this service. The server is 
 # accessed via repo.messaging.hostname and repo.messaging.port 
@@ -205,7 +202,6 @@ pit.security.allowedOriginPattern=http*://localhost:[*]
 # The time in minutes after which Entries will expire, starting from the
 # last update.
 # pit.typeregistry.cache.lifetimeMinutes:10
-
 # Profiles may disallow additional attributes in the PID records. This
 # option may be used to override this behavior for this instance.
 # If set to false, it will behave as the profiles describe.
@@ -224,7 +220,6 @@ pit.validation.alwaysAllowAdditionalAttributes=true
 #######################################################
 #################### PID GENERATOR ####################
 #######################################################
-
 # The PID generator to use for the suffix. Possible values:
 # "uuid4": generates a UUID v4 (random) PID suffix.
 # "hex-chunks": generates hex-chunks. Each chunk is four characters long. Example: 1D6C-152C-C9E0-C136-1509
@@ -256,7 +251,6 @@ pit.pidgeneration.casing=lower
 #       This means that you can not create PIDs with a suffix starting with the system prefix.
 #       Example: system prefix="abc", suffix=abcdef
 #              => PID="abc/def" (delimiter may depend on PID system)
-
 # pit.pidgeneration.custom-client-pids-enabled=false
 ### DANGEROUS OPTIONS! Please read carefully! ########################################
 
@@ -269,13 +263,11 @@ pit.pidgeneration.casing=lower
 ###   system is set to LOCAL ###
 ### - Required for messaging ###
 ################################
-
 # This database will always run, as it is also required for the messaging feature,
 # but for the messaging it is not required to be persistent.
 # But the service will also use this database to store known PIDs.
 # This can be used as a backup or documentation of all PIDs.
 # The following properties can (and should) be set.
-
 # When to store PIDs in the local database ("known PIDs")
 pit.storage.strategy=keep-resolved-and-modified
 
@@ -289,10 +281,143 @@ spring.datasource.driver-class-name=org.h2.Driver
 # WARNING: Change the DB to be stored somewhere outside of /tmp!
 spring.datasource.url=jdbc:h2:file:/tmp/database;MODE=LEGACY;NON_KEYWORDS=VALUE
 
+
 # Credentials for the database:
 spring.datasource.username=typid
 spring.datasource.password=secure_me
-
 # Do not change ddl-auto if you do not know what you are doing:
 # https://docs.spring.io/spring-boot/docs/1.1.0.M1/reference/html/howto-database-initialization.html
 spring.jpa.hibernate.ddl-auto=update
+
+
+################################
+####### Observability ##########
+################################
+
+### General OpenTelemetry Configuration ###
+# Allows unrestricted access to Prometheus metrics endpoint
+management.endpoint.prometheus.access=unrestricted
+# Enables automatic OpenTelemetry SDK configuration
+otel.java.global-autoconfigure.enabled=true
+# Integrates OpenTelemetry with Micrometer metrics
+otel.instrumentation.micrometer.enabled=true
+# Service name appearing in telemetry data
+otel.service.name=${spring.application.name}
+# OpenTelemetry Collector endpoint URL
+otel.exporter.otlp.endpoint=http://localhost:4318
+# Protocol for telemetry data export (recommended for performance)
+otel.exporter.otlp.protocol=http/protobuf
+# Context propagation formats (W3C standards)
+otel.propagators=tracecontext,baggage
+
+### OpenTelemetry Metrics Configuration ###
+# Export metrics via OTLP protocol
+otel.metrics.exporter=otlp
+# Enable OTLP metrics export through management endpoints
+management.otlp.metrics.export.enabled=true
+# Metrics endpoint URL
+management.otlp.metrics.export.url=http://localhost:4318/v1/metrics
+# Metrics export interval (production-appropriate)
+management.metrics.export.defaults.step=15s
+# HTTP request duration histogram buckets for SLA monitoring
+management.metrics.distribution.sla.http.server.requests=1ms,10ms,50ms,100ms,200ms,500ms,1s,2s,5s
+# Enable percentile histograms for timer metrics
+management.metrics.distribution.percentiles-histogram[timer]=true
+# Timer metrics histogram buckets (database queries, method execution)
+management.metrics.distribution.sla[timer]=0.1ms,0.5ms,1ms,10ms,50ms,100ms,200ms,500ms,1s,2s,5s
+# Enable percentile histograms for HTTP request metrics
+management.metrics.distribution.percentiles-histogram.http.server.requests=true
+# Tag all metrics with service name for filtering
+management.metrics.tags.service_name=${spring.application.name}
+# Tag all metrics with environment for separation
+management.metrics.tags.environment=${spring.profiles.active}
+# Disable Prometheus export (using OTLP instead)
+management.prometheus.metrics.export.enabled=false
+
+### OpenTelemetry Logging Configuration ###
+# Enable log export via OTLP
+management.otlp.logging.export.enabled=true
+# Logs endpoint URL
+management.otlp.logging.endpoint=http://localhost:4318/v1/logs
+# Capture trace/span IDs in logs for correlation
+otel.instrumentation.log4j-appender.experimental.capture-mdc-attributes=trace_id,span_id
+# Enable logging context propagation
+logging.context.enabled=true
+
+### Tracing Configuration ###
+# Production-safe sampling rate (10% of requests)
+management.tracing.sampling.probability=0.1
+# Traces endpoint URL
+management.otlp.tracing.endpoint=http://localhost:4318/v1/traces
+# Record HTTP exchanges for detailed analysis
+management.httpexchanges.recording.enabled=true
+# Enable baggage correlation for distributed traces
+management.tracing.baggage.correlation.enabled=true
+# Enable @Observed annotation support
+management.observations.annotations.enabled=true
+# Enable all built-in Spring Boot observations
+management.observations.enable.all=true
+# Enable runtime telemetry (JVM metrics)
+otel.instrumentation.runtime-telemetry.enabled=true
+# (optional: enable JFR/Java17-specific telemetry)
+otel.instrumentation.runtime-telemetry-java17.enabled=true
+# Enable Spring WebMVC request tracing
+otel.instrumentation.spring-webmvc.enabled=true
+# Enable OpenTelemetry annotation support (@WithSpan)
+otel.instrumentation.opentelemetry-instrumentation-annotations.enabled=true
+# Enable OpenTelemetry annotation support (@WithSpan)
+otel.instrumentation.annotations.enabled=true
+# Parent-based sampling with trace ID ratio
+otel.traces.sampler=parentbased_traceidratio
+# Sampling ratio argument (10%)
+otel.traces.sampler.arg=0.1
+# Enable Spring Boot application lifecycle tracing
+otel.instrumentation.spring-boot.enabled=true
+# Enable Spring Data repository tracing
+otel.instrumentation.spring-data.enabled=true
+# Enable RabbitMQ message tracing
+otel.instrumentation.spring-rabbit.enabled=true
+# Enable Spring Security tracing
+otel.instrumentation.spring-security.enabled=true
+
+### 5) Sensitive Information Configuration (COMMENTED OUT FOR PRODUCTION) ###
+## WARNING: These capture potentially sensitive data - NOT for production
+## Captures HTTP request headers (may contain auth tokens)
+# otel.instrumentation.http.client.capture-request-headers=true
+## Captures HTTP response headers (may contain sensitive data)
+# otel.instrumentation.http.client.capture-response-headers=true
+## Disables URL parameter redaction (may expose sensitive URLs)
+# otel.instrumentation.http.client.experimental.redact-query-parameters=false
+## Include PII (Personally Identifiable Information) as trace attributes
+pit.observability.includePiiInTraces=true
+
+### 6) Experimental/Verbose Configuration (COMMENTED OUT FOR PRODUCTION) ###
+## WARNING: Experimental features or high overhead - NOT for production
+## Too frequent export (high overhead)
+# management.otlp.metrics.export.step=2s
+## 100% sampling (performance impact)
+# management.tracing.sampling.probability=1.0
+## Exports unsampled traces (storage cost)
+# management.tracing.opentelemetry.export.include-unsampled=true
+## 100% sampling argument
+# otel.traces.sampler.arg=1
+## Verbose log attributes (experimental)
+# otel.instrumentation.logback-appender.experimental-log-attributes=true
+## Code location capture (experimental and expensive)
+# otel.instrumentation.logback-appender.experimental.capture-code-attributes=true
+## Log marker capture (experimental feature)
+# otel.instrumentation.logback-appender.experimental.capture-marker-attribute=true
+## Experimental HTTP telemetry (may be unstable)
+# otel.instrumentation.http.client.emit-experimental-telemetry=true
+## Experimental JDBC tracing (may impact performance)
+# otel.instrumentation.jdbc.experimental.transaction.enabled=true
+## Request parameter capture (may expose sensitive data)
+# otel.instrumentation.spring-webmvc.experimental.capture-request-parameters=true
+## Verbose controller telemetry (experimental)
+# otel.instrumentation.spring-webmvc.experimental.capture-controller-telemetry=true
+## View rendering telemetry (experimental and verbose)
+# otel.instrumentation.spring-webmvc.experimental.capture-view-telemetry=true
+## Hibernate span control (experimental, may miss important data)
+# otel.instrumentation.hibernate.experimental.span-suppression-strategy=statement-only
+## Verbose JPA query reporting (experimental, performance impact)
+# otel.instrumentation.jpa.experimental.query-reporting=true