Skip to content

jmx-scraper implement stable service.instance.id #2270

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
breedx-splk merged 9 commits into from
Sep 23, 2025
Merged

jmx-scraper implement stable service.instance.id #2270

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
7d6e440
move otel init after scraper config
SylvainJuge Sep 17, 2025
f46cf5c
implement stable service ID
SylvainJuge Sep 17, 2025
cd6143e
Merge branch 'main' of github.com:open-telemetry/opentelemetry-java-c…
SylvainJuge Sep 17, 2025
614ce54
fix tests
SylvainJuge Sep 17, 2025
12acf64
implement test based on scraper logs
SylvainJuge Sep 17, 2025
04c675e
use semconv constant for service ID
SylvainJuge Sep 18, 2025
3fdb909
Merge branch 'main' of github.com:open-telemetry/opentelemetry-java-c…
SylvainJuge Sep 19, 2025
48bbe80
post-review changes
SylvainJuge Sep 19, 2025
487ca2d
post-review changes
SylvainJuge Sep 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions jmx-scraper/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ dependencies {

implementation("io.opentelemetry.instrumentation:opentelemetry-jmx-metrics")

implementation("io.opentelemetry.semconv:opentelemetry-semconv-incubating")

testImplementation("org.junit-pioneer:junit-pioneer")
testImplementation("io.opentelemetry:opentelemetry-sdk-testing")
testImplementation("org.awaitility:awaitility")
Expand Down
39 changes: 38 additions & 1 deletion ...raper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/JmxConnectionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

import java.nio.file.Path;
import java.security.cert.X509Certificate;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import org.junit.jupiter.api.AfterAll;
Expand Down Expand Up @@ -131,7 +132,7 @@ private void testServerSsl(
}

@ParameterizedTest
@EnumSource(value = JmxScraperContainer.ConfigSource.class)
@EnumSource
void serverSslClientSsl(JmxScraperContainer.ConfigSource configSource) {
// Note: this could have been made simpler by relying on the fact that keystore could be used
// as a trust store, but having clear split provides also some extra clarity
Expand Down Expand Up @@ -175,6 +176,42 @@ void serverSslClientSsl(JmxScraperContainer.ConfigSource configSource) {
.withConfigSource(configSource));
}

@Test
void stableServiceInstanceServiceId() {
// start a single app, connect twice to it and check that the service id is the same
try (TestAppContainer app = appContainer().withJmxPort(JMX_PORT)) {
app.start();

UUID firstId = startScraperAndGetServiceId();
UUID secondId = startScraperAndGetServiceId();

assertThat(firstId)
.describedAs(
"connecting twice to the same JVM should return the same service instance ID")
.isEqualTo(secondId);
}
}

private static UUID startScraperAndGetServiceId() {
try (JmxScraperContainer scraper =
scraperContainer()
.withRmiServiceUrl(APP_HOST, JMX_PORT)
// does not need to be tested on all config sources
.withConfigSource(JmxScraperContainer.ConfigSource.SYSTEM_PROPERTIES)) {
scraper.start();
waitTerminated(scraper);
String[] logLines = scraper.getLogs().split("\n");
UUID serviceId = null;
for (String logLine : logLines) {
if (logLine.contains("remote service instance ID")) {
serviceId = UUID.fromString(logLine.substring(logLine.lastIndexOf(":") + 1).trim());
}
}
assertThat(serviceId).describedAs("unable to get service instance ID from logs").isNotNull();
return serviceId;
}
}

private static void connectionTest(
Function<TestAppContainer, TestAppContainer> customizeApp,
Function<JmxScraperContainer, JmxScraperContainer> customizeScraper) {
Expand Down
88 changes: 62 additions & 26 deletions jmx-scraper/src/main/java/io/opentelemetry/contrib/jmxscraper/JmxScraper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,14 @@

package io.opentelemetry.contrib.jmxscraper;

import static io.opentelemetry.semconv.incubating.ServiceIncubatingAttributes.SERVICE_INSTANCE_ID;
import static java.util.Arrays.asList;
import static java.util.Collections.singletonList;
import static java.util.Optional.ofNullable;
import static java.util.logging.Level.INFO;
import static java.util.logging.Level.SEVERE;

import io.opentelemetry.api.GlobalOpenTelemetry;
import io.opentelemetry.api.OpenTelemetry;
import io.opentelemetry.api.common.Attributes;
import io.opentelemetry.contrib.jmxscraper.config.JmxScraperConfig;
import io.opentelemetry.contrib.jmxscraper.config.PropertiesCustomizer;
import io.opentelemetry.contrib.jmxscraper.config.PropertiesSupplier;
Expand All @@ -20,22 +21,30 @@
import io.opentelemetry.instrumentation.jmx.yaml.RuleParser;
import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk;
import io.opentelemetry.sdk.autoconfigure.spi.ConfigurationException;
import io.opentelemetry.sdk.resources.Resource;
import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.annotation.Nullable;
import javax.management.MBeanServerConnection;
import javax.management.ObjectName;
import javax.management.remote.JMXConnector;

public final class JmxScraper {

private static final Logger logger = Logger.getLogger(JmxScraper.class.getName());
private static final String CONFIG_ARG = "-config";
private static final String TEST_ARG = "-test";
Expand Down Expand Up @@ -64,36 +73,40 @@ public static void main(String[] args) {
Properties argsConfig = argsToConfig(effectiveArgs);
propagateToSystemProperties(argsConfig);

// auto-configure and register SDK
PropertiesCustomizer configCustomizer = new PropertiesCustomizer();
AutoConfiguredOpenTelemetrySdk.builder()
.addPropertiesSupplier(new PropertiesSupplier(argsConfig))
.addPropertiesCustomizer(configCustomizer)
.setResultAsGlobal()
.build();

// auto-configure SDK
OpenTelemetry openTelemetry =
AutoConfiguredOpenTelemetrySdk.builder()
.addPropertiesSupplier(new PropertiesSupplier(argsConfig))
.addPropertiesCustomizer(configCustomizer)
// we rely on the config customizer to be executed first to get effective config
.addResourceCustomizer(
(resource, configProperties) -> {
UUID instanceId =
getRemoteServiceInstanceId(configCustomizer.getConnectorBuilder());
if (resource.getAttribute(SERVICE_INSTANCE_ID) != null || instanceId == null) {
return resource;
}
logger.log(Level.INFO, "remote service instance ID: " + instanceId);
return resource.merge(
Resource.create(Attributes.of(SERVICE_INSTANCE_ID, instanceId.toString())));
})
.build()
.getOpenTelemetrySdk();

// scraper configuration and connector builder are built using effective SDK configuration
// thus we have to get it after the SDK is built
JmxScraperConfig scraperConfig = configCustomizer.getScraperConfig();

long exportSeconds = scraperConfig.getSamplingInterval().toMillis() / 1000;
logger.log(INFO, "metrics export interval (seconds) = " + exportSeconds);

JmxMetricInsight service =
JmxMetricInsight.createService(
GlobalOpenTelemetry.get(), scraperConfig.getSamplingInterval().toMillis());
JmxConnectorBuilder connectorBuilder =
JmxConnectorBuilder.createNew(scraperConfig.getServiceUrl());

ofNullable(scraperConfig.getUsername()).ifPresent(connectorBuilder::withUser);
ofNullable(scraperConfig.getPassword()).ifPresent(connectorBuilder::withPassword);

if (scraperConfig.isRegistrySsl()) {
connectorBuilder.withSslRegistry();
}
JmxConnectorBuilder connectorBuilder = configCustomizer.getConnectorBuilder();

if (testMode) {
System.exit(testConnection(connectorBuilder) ? 0 : 1);
} else {
JmxScraper jmxScraper = new JmxScraper(connectorBuilder, service, scraperConfig);
JmxMetricInsight jmxInsight =
JmxMetricInsight.createService(
openTelemetry, scraperConfig.getSamplingInterval().toMillis());
JmxScraper jmxScraper = new JmxScraper(connectorBuilder, jmxInsight, scraperConfig);
jmxScraper.start();
}
} catch (ConfigurationException e) {
Expand All @@ -117,7 +130,6 @@ public static void main(String[] args) {

private static boolean testConnection(JmxConnectorBuilder connectorBuilder) {
try (JMXConnector connector = connectorBuilder.build()) {

MBeanServerConnection connection = connector.getMBeanServerConnection();
Integer mbeanCount = connection.getMBeanCount();
if (mbeanCount > 0) {
Expand All @@ -133,6 +145,30 @@ private static boolean testConnection(JmxConnectorBuilder connectorBuilder) {
}
}

@Nullable
static UUID getRemoteServiceInstanceId(JmxConnectorBuilder connectorBuilder) {
try (JMXConnector jmxConnector = connectorBuilder.build()) {
MBeanServerConnection connection = jmxConnector.getMBeanServerConnection();

StringBuilder id = new StringBuilder();
try {
ObjectName objectName = new ObjectName("java.lang:type=Runtime");
for (String attribute : Arrays.asList("StartTime", "Name")) {
Object value = connection.getAttribute(objectName, attribute);
if (id.length() > 0) {
id.append(" ");
}
id.append(value);
}
Comment on lines +157 to +164
Copy link
Contributor

@breedx-splk breedx-splk Sep 18, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The description in #2206 includes the Pid but it's missing here. Is that intentional? Maybe the start time is enough....but I like the idea of including the pid, because time might introduce an edge case.

Copy link
Contributor Author

@SylvainJuge SylvainJuge Sep 19, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Including the Pid is the first thing that failed when testing with Java8, so it must have been added in a later JVM version. From what I've seen so far it's implicitly included in the Name attribute which is usually in the 1234@hostname format, where 1234 is the PID.

return UUID.nameUUIDFromBytes(id.toString().getBytes(StandardCharsets.UTF_8));
} catch (Exception e) {
throw new IllegalStateException(e);
}
} catch (IOException e) {
return null;
}
}

// package private for testing
static void propagateToSystemProperties(Properties properties) {
for (Map.Entry<Object, Object> entry : properties.entrySet()) {
Expand Down
30 changes: 30 additions & 0 deletions ...craper/src/main/java/io/opentelemetry/contrib/jmxscraper/config/PropertiesCustomizer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,13 @@
import static io.opentelemetry.contrib.jmxscraper.config.JmxScraperConfig.JMX_INTERVAL_LEGACY;
import static io.opentelemetry.contrib.jmxscraper.config.JmxScraperConfig.METRIC_EXPORT_INTERVAL;

import io.opentelemetry.contrib.jmxscraper.JmxConnectorBuilder;
import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.annotation.Nullable;

Expand All @@ -24,6 +27,8 @@ public final class PropertiesCustomizer implements Function<ConfigProperties, Ma

@Nullable private JmxScraperConfig scraperConfig;

@Nullable private JmxConnectorBuilder connectorBuilder;

@Override
public Map<String, String> apply(ConfigProperties config) {
Map<String, String> result = new HashMap<>();
Expand All @@ -44,10 +49,28 @@ public Map<String, String> apply(ConfigProperties config) {
result.put(METRIC_EXPORT_INTERVAL, intervalLegacy + "ms");
}

// scraper config and connector builder must be initialized with the effective SDK configuration
// thus we need to initialize them here and then rely on getter being called after this method.
scraperConfig = JmxScraperConfig.fromConfig(config);
connectorBuilder = createConnectorBuilder(scraperConfig);

long exportSeconds = scraperConfig.getSamplingInterval().toMillis() / 1000;
logger.log(Level.INFO, "metrics export interval (seconds) = " + exportSeconds);

return result;
}

private static JmxConnectorBuilder createConnectorBuilder(JmxScraperConfig scraperConfig) {
JmxConnectorBuilder connectorBuilder =
JmxConnectorBuilder.createNew(scraperConfig.getServiceUrl());
Optional.ofNullable(scraperConfig.getUsername()).ifPresent(connectorBuilder::withUser);
Optional.ofNullable(scraperConfig.getPassword()).ifPresent(connectorBuilder::withPassword);
if (scraperConfig.isRegistrySsl()) {
connectorBuilder.withSslRegistry();
}
return connectorBuilder;
}

/**
* Get scraper configuration from the previous call to {@link #apply(ConfigProperties)}
*
Expand All @@ -60,4 +83,11 @@ public JmxScraperConfig getScraperConfig() {
}
return scraperConfig;
}

public JmxConnectorBuilder getConnectorBuilder() {
if (connectorBuilder == null) {
throw new IllegalStateException("apply() must be called before getConnectorBuilder()");
}
return connectorBuilder;
}
}
16 changes: 10 additions & 6 deletions ...er/src/test/java/io/opentelemetry/contrib/jmxscraper/config/PropertiesCustomizerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,20 @@

class PropertiesCustomizerTest {

private static final String DUMMY_URL = "service:jmx:rmi:///jndi/rmi://host:999/jmxrmi";
Copy link
Contributor Author

@SylvainJuge SylvainJuge Sep 18, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[for reviewer] we now have to test with an URL that can be parsed because the parsing is triggered, while it used to not be parsed previously.


@Test
void tryGetConfigBeforeApply() {
void tryGetBeforeApply() {
assertThatThrownBy(() -> new PropertiesCustomizer().getScraperConfig())
.isInstanceOf(IllegalStateException.class);
assertThatThrownBy(() -> new PropertiesCustomizer().getConnectorBuilder())
.isInstanceOf(IllegalStateException.class);
}

@Test
void defaultOtlpExporter() {
Map<String, String> map = new HashMap<>();
map.put("otel.jmx.service.url", "dummy-url");
map.put("otel.jmx.service.url", DUMMY_URL);
map.put("otel.jmx.target.system", "jvm");
ConfigProperties config = DefaultConfigProperties.createFromMap(map);

Expand All @@ -37,7 +41,7 @@ void defaultOtlpExporter() {
@Test
void explicitExporterSet() {
Map<String, String> map = new HashMap<>();
map.put("otel.jmx.service.url", "dummy-url");
map.put("otel.jmx.service.url", DUMMY_URL);
map.put("otel.jmx.target.system", "jvm");
map.put("otel.metrics.exporter", "otlp,logging");
ConfigProperties config = DefaultConfigProperties.createFromMap(map);
Expand All @@ -49,7 +53,7 @@ void explicitExporterSet() {
@Test
void getSomeConfiguration() {
Map<String, String> map = new HashMap<>();
map.put("otel.jmx.service.url", "dummy-url");
map.put("otel.jmx.service.url", DUMMY_URL);
map.put("otel.jmx.target.system", "jvm");
map.put("otel.metrics.exporter", "otlp");
ConfigProperties config = DefaultConfigProperties.createFromMap(map);
Expand All @@ -67,7 +71,7 @@ void getSomeConfiguration() {
@Test
void setSdkMetricExportFromJmxInterval() {
Map<String, String> map = new HashMap<>();
map.put("otel.jmx.service.url", "dummy-url");
map.put("otel.jmx.service.url", DUMMY_URL);
map.put("otel.jmx.target.system", "jvm");
map.put("otel.metrics.exporter", "otlp");
map.put("otel.jmx.interval.milliseconds", "10000");
Expand All @@ -83,7 +87,7 @@ void setSdkMetricExportFromJmxInterval() {
@Test
void sdkMetricExportIntervalPriority() {
Map<String, String> map = new HashMap<>();
map.put("otel.jmx.service.url", "dummy-url");
map.put("otel.jmx.service.url", DUMMY_URL);
map.put("otel.jmx.target.system", "jvm");
map.put("otel.metrics.exporter", "otlp");
map.put("otel.jmx.interval.milliseconds", "10000");
Expand Down
Loading