add support for attribute[*] with capture_jmx_metrics (#3376)

* add support for attribute[*] with capture_jmx_metrics

* fixx doc and remove comment

* expand wildcard import, refactor wildcard check, format better

* add changelog entry for attribute[*]

* fix changelog typo

* add 'from' version for feature

Author: jackshirazi

CHANGELOG.asciidoc

@@ -37,6 +37,7 @@ Use subheadings with the "=====" level for adding notes for unreleased changes:
 * Added support for AWS SDK 2.21 - {pull}3373[#3373]
 * Capture bucket and object key to Lambda transaction as OTel attributes - `aws.s3.bueckt`, `aws.s3.key` - {pull}3364[#3364]
 * Added `context_propagation_only` configuration option - {pull}3358[#3358]
+* Added attribute[*] for JMX pattern metrics (all metrics can now be generated with `object_name[*:type=*,name=*] attribute[*]`) - {pull}3376[#3376]
 
 [float]
 ===== Bug fixes

apm-agent-plugins/apm-jmx-plugin/src/main/java/co/elastic/apm/agent/jmx/JmxConfiguration.java

@@ -58,8 +58,9 @@ public class JmxConfiguration extends ConfigurationOptionProvider {
             "\n" +
             "The agent creates `labels` for each link:https://docs.oracle.com/javase/7/docs/api/javax/management/ObjectName.html#getKeyPropertyList()[JMX key property] such as `type` and `name`.\n" +
             "\n" +
-            "The link:https://docs.oracle.com/javase/7/docs/api/javax/management/ObjectName.html[JMX object name pattern] supports wildcards.\n" +
-            "In this example, the agent will create a metricset for each memory pool `name` (such as `G1 Old Generation` and `G1 Young Generation`)\n" +
+            "The link:https://docs.oracle.com/javase/7/docs/api/javax/management/ObjectName.html[JMX object name pattern] supports wildcards. The attribute definition does NOT support wildcards, but a special definition `attribute[*]` is accepted (from 1.44.0) to mean match all possible (numeric) attributes for the associated object name pattern\n" +
+            "The definition `object_name[*:type=*,name=*] attribute[*]` would match all possible JMX metrics\n"  +
+            "In the following example, the agent will create a metricset for each memory pool `name` (such as `G1 Old Generation` and `G1 Young Generation`)\n" +
             "\n" +
             "----\n" +
             "object_name[java.lang:type=GarbageCollector,name=*] attribute[CollectionCount:metric_name=collection_count] attribute[CollectionTime]\n" +

apm-agent-plugins/apm-jmx-plugin/src/main/java/co/elastic/apm/agent/jmx/JmxMetric.java

@@ -184,8 +184,12 @@ public String getJmxAttributeName() {
             return jmxAttributeName;
         }
 
-        public String getMetricName() {
-            return JMX_PREFIX + (metricName != null ? metricName : jmxAttributeName);
+        public String getMetricName(String foundAttributeName) {
+            if (!foundAttributeName.isEmpty() && "*".equals(jmxAttributeName)) {
+                return JMX_PREFIX + (metricName != null ? metricName : foundAttributeName);
+            } else {
+                return JMX_PREFIX + (metricName != null ? metricName : jmxAttributeName);
+            }
         }
 
         @Override
@@ -211,8 +215,8 @@ public Labels getLabels(ObjectName objectName) {
             return Labels.Mutable.of(objectName.getKeyPropertyList());
         }
 
-        public String getCompositeMetricName(String key) {
-            return getMetricName() + "." + key;
+        public String getCompositeMetricName(String key, String foundAttributeName) {
+            return getMetricName(foundAttributeName) + "." + key;
         }
     }
 

apm-agent-plugins/apm-jmx-plugin/src/main/java/co/elastic/apm/agent/jmx/JmxMetricTracker.java

@@ -33,6 +33,8 @@
 import javax.management.AttributeNotFoundException;
 import javax.management.InstanceNotFoundException;
 import javax.management.JMException;
+import javax.management.MBeanAttributeInfo;
+import javax.management.MBeanInfo;
 import javax.management.MBeanServer;
 import javax.management.MBeanServerDelegate;
 import javax.management.MBeanServerFactory;
@@ -42,6 +44,7 @@
 import javax.management.NotificationListener;
 import javax.management.ObjectInstance;
 import javax.management.ObjectName;
+import javax.management.RuntimeMBeanException;
 import javax.management.openmbean.CompositeData;
 import javax.management.relation.MBeanServerNotificationFilter;
 import java.lang.management.ManagementFactory;
@@ -284,7 +287,7 @@ private void register(List<JmxMetric> jmxMetrics, MBeanServer server) {
     }
 
     /**
-     * A single {@link JmxMetric} can yield multiple {@link JmxMetricRegistration}s if the {@link JmxMetric} contains multiple {@link JmxMetric#attributes}
+     * A single {@link JmxMetric} can yield multiple {@link JmxMetricRegistration}s if the {@link JmxMetric} contains multiple attributes
      */
     private List<JmxMetricRegistration> compileJmxMetricRegistrations(List<JmxMetric> jmxMetrics, MBeanServer server) {
         List<JmxMetricRegistration> registrations = new ArrayList<>();
@@ -308,40 +311,98 @@ private static void addJmxMetricRegistration(final JmxMetric jmxMetric, List<Jmx
         for (ObjectInstance mbean : mbeans) {
             for (JmxMetric.Attribute attribute : jmxMetric.getAttributes()) {
                 final ObjectName objectName = mbean.getObjectName();
-                final Object value;
-                try {
-                    value = server.getAttribute(objectName, attribute.getJmxAttributeName());
-                    if (value instanceof Number) {
-                        logger.debug("Found number attribute {}={}", attribute.getJmxAttributeName(), value);
-                        registrations.add(new JmxMetricRegistration(attribute.getMetricName(),
-                            attribute.getLabels(objectName),
-                            attribute.getJmxAttributeName(),
-                            null,
-                            objectName));
-                    } else if (value instanceof CompositeData) {
-                        final CompositeData compositeValue = (CompositeData) value;
-                        for (final String key : compositeValue.getCompositeType().keySet()) {
-                            if (compositeValue.get(key) instanceof Number) {
-                                logger.debug("Found composite number attribute {}.{}={}", attribute.getJmxAttributeName(), key, value);
-                                registrations.add(new JmxMetricRegistration(attribute.getCompositeMetricName(key),
-                                    attribute.getLabels(objectName),
-                                    attribute.getJmxAttributeName(),
-                                    key,
-                                    objectName));
+                final String metricPrepend = metricPrepend(attribute.getLabels(objectName));
+                if (isWildcard(attribute)) {
+                    MBeanInfo info = server.getMBeanInfo(objectName);
+                    MBeanAttributeInfo[] attrInfo = info.getAttributes();
+                    for (MBeanAttributeInfo attr : attrInfo) {
+                        try {
+                            final Object value = server.getAttribute(objectName, attr.getName());
+                            addJmxMetricRegistration(jmxMetric, registrations, objectName, value, attribute, attr.getName(), metricPrepend);
+                        } catch (AttributeNotFoundException e) {
+                            logger.warn("Can't create metric '{}' because attribute '{}' could not be found", jmxMetric, attribute.getJmxAttributeName());
+                        } catch (RuntimeMBeanException e) {
+                            if (e.getCause() instanceof UnsupportedOperationException) {
+                                //ignore this attribute
                             } else {
-                                logger.warn("Can't create metric '{}' because composite value '{}' is not a number: '{}'", jmxMetric, key, value);
+                                throw e;
                             }
                         }
-                    } else {
-                        logger.warn("Can't create metric '{}' because attribute '{}' is not a number: '{}'", jmxMetric, attribute.getJmxAttributeName(), value);
                     }
-                } catch (AttributeNotFoundException e) {
-                    logger.warn("Can't create metric '{}' because attribute '{}' could not be found", jmxMetric, attribute.getJmxAttributeName());
+                } else {
+                    final Object value = server.getAttribute(objectName, attribute.getJmxAttributeName());
+                    try {
+                        addJmxMetricRegistration(jmxMetric, registrations, objectName, value, attribute, attribute.getJmxAttributeName(), null);
+                    } catch (AttributeNotFoundException e) {
+                        logger.warn("Can't create metric '{}' because attribute '{}' could not be found", jmxMetric, attribute.getJmxAttributeName());
+                    }
                 }
             }
         }
     }
 
+    private static boolean isWildcard(JmxMetric.Attribute attribute) {
+        return "*".equals(attribute.getJmxAttributeName());
+    }
+
+    private static String metricPrepend(Labels labels) {
+        List<String> keys = labels.getKeys();
+        for (int i = 0; i < keys.size(); i++) {
+            if ("type".equals(keys.get(i))) {
+                return labels.getValue(i) + ".";
+            }
+        }
+        return "";
+    }
+
+    private static void addJmxMetricRegistration(JmxMetric jmxMetric, List<JmxMetricRegistration> registrations, ObjectName objectName, Object value, JmxMetric.Attribute attribute, String attributeName, String metricPrepend) throws AttributeNotFoundException {
+        if (value instanceof Number) {
+            logger.debug("Found number attribute {}={}", attribute.getJmxAttributeName(), value);
+            registrations.add(
+                new JmxMetricRegistration(
+                    attribute.getMetricName(
+                        metricPrepend == null ?
+                            attributeName :
+                            metricPrepend + attributeName
+                    ),
+                    attribute.getLabels(objectName),
+                    attributeName,
+                    null,
+                    objectName
+                )
+            );
+        } else if (value instanceof CompositeData) {
+            final CompositeData compositeValue = (CompositeData) value;
+            for (final String key : compositeValue.getCompositeType().keySet()) {
+                if (compositeValue.get(key) instanceof Number) {
+                    logger.debug("Found composite number attribute {}.{}={}", attribute.getJmxAttributeName(), key, value);
+                    registrations.add(
+                        new JmxMetricRegistration(
+                            attribute.getCompositeMetricName(
+                                key,
+                                metricPrepend == null ?
+                                    attributeName :
+                                    metricPrepend + attributeName
+                            ),
+                            attribute.getLabels(objectName),
+                            attributeName,
+                            key,
+                            objectName
+                        )
+                    );
+                } else {
+                    if (!isWildcard(attribute)) {
+                        logger.warn("Can't create metric '{}' because composite value '{}' is not a number: '{}'", jmxMetric, key, value);
+                    }
+                }
+            }
+        } else {
+            if (!isWildcard(attribute)) {
+                logger.warn("Can't create metric '{}' because attribute '{}' is not a number: '{}'", jmxMetric, attributeName, value);
+            }
+        }
+    }
+
     static class JmxMetricRegistration {
         private static final Logger logger = LoggerFactory.getLogger(JmxMetricRegistration.class);
         private final String metricName;

apm-agent-plugins/apm-jmx-plugin/src/test/java/co/elastic/apm/agent/jmx/JmxMetricTrackerTest.java

@@ -83,6 +83,17 @@ void testGC() throws Exception {
         printMetricSets();
     }
 
+    @Test
+    void testAttributeWildcard() throws Exception {
+        setConfig(JmxMetric.valueOf("object_name[java.lang:type=GarbageCollector,name=*] attribute[*]"));
+        for (GarbageCollectorMXBean gcBean : ManagementFactory.getGarbageCollectorMXBeans()) {
+            String memoryManagerName = gcBean.getName();
+            assertThat(metricRegistry.getGaugeValue("jvm.jmx.collection_count", Labels.Mutable.of("name", memoryManagerName).add("type", "GarbageCollector"))).isNotNegative();
+            assertThat(metricRegistry.getGaugeValue("jvm.jmx.CollectionTime", Labels.Mutable.of("name", memoryManagerName).add("type", "GarbageCollector"))).isNotNegative();
+        }
+        printMetricSets();
+    }
+
     @Test
     void testRemoveMetric() throws Exception {
         setConfig(JmxMetric.valueOf("object_name[java.lang:type=GarbageCollector,name=*] attribute[CollectionCount:metric_name=collection_count] attribute[CollectionTime]"));

apm-agent-plugins/apm-jmx-plugin/src/test/java/co/elastic/apm/agent/jmx/JmxMetricValueConverterTest.java

@@ -40,7 +40,7 @@ private static void testMetricName(String s, String expectedMetricName, String e
         JmxMetric metric = JmxMetric.valueOf(s);
         assertThat(metric.getAttributes()).hasSize(1);
         metric.getAttributes().forEach(a -> {
-            assertThat(a.getMetricName()).isEqualTo(expectedMetricName);
+            assertThat(a.getMetricName("")).isEqualTo(expectedMetricName);
             assertThat(a.getJmxAttributeName()).isEqualTo(expectedJmxAttributeName);
         });
     }

docs/configuration.asciidoc

@@ -2030,8 +2030,9 @@ Note that all JMX metric names will be prefixed with `jvm.jmx.` by the agent.
 
 The agent creates `labels` for each link:https://docs.oracle.com/javase/7/docs/api/javax/management/ObjectName.html#getKeyPropertyList()[JMX key property] such as `type` and `name`.
 
-The link:https://docs.oracle.com/javase/7/docs/api/javax/management/ObjectName.html[JMX object name pattern] supports wildcards.
-In this example, the agent will create a metricset for each memory pool `name` (such as `G1 Old Generation` and `G1 Young Generation`)
+The link:https://docs.oracle.com/javase/7/docs/api/javax/management/ObjectName.html[JMX object name pattern] supports wildcards. The attribute definition does NOT support wildcards, but a special definition `attribute[*]` is accepted (from 1.44.0) to mean match all possible (numeric) attributes for the associated object name pattern
+The definition `object_name[*:type=*,name=*] attribute[*]` would match all possible JMX metrics
+In the following example, the agent will create a metricset for each memory pool `name` (such as `G1 Old Generation` and `G1 Young Generation`)
 
 ----
 object_name[java.lang:type=GarbageCollector,name=*] attribute[CollectionCount:metric_name=collection_count] attribute[CollectionTime]
@@ -4326,8 +4327,9 @@ Example: `5ms`.
 # 
 # The agent creates `labels` for each link:https://docs.oracle.com/javase/7/docs/api/javax/management/ObjectName.html#getKeyPropertyList()[JMX key property] such as `type` and `name`.
 # 
-# The link:https://docs.oracle.com/javase/7/docs/api/javax/management/ObjectName.html[JMX object name pattern] supports wildcards.
-# In this example, the agent will create a metricset for each memory pool `name` (such as `G1 Old Generation` and `G1 Young Generation`)
+# The link:https://docs.oracle.com/javase/7/docs/api/javax/management/ObjectName.html[JMX object name pattern] supports wildcards. The attribute definition does NOT support wildcards, but a special definition `attribute[*]` is accepted (from 1.44.0) to mean match all possible (numeric) attributes for the associated object name pattern
+# The definition `object_name[*:type=*,name=*] attribute[*]` would match all possible JMX metrics
+# In the following example, the agent will create a metricset for each memory pool `name` (such as `G1 Old Generation` and `G1 Young Generation`)
 # 
 # ----
 # object_name[java.lang:type=GarbageCollector,name=*] attribute[CollectionCount:metric_name=collection_count] attribute[CollectionTime]