diff --git a/common/src/main/java/com/zegelin/cassandra/exporter/FactoriesSupplier.java b/common/src/main/java/com/zegelin/cassandra/exporter/FactoriesSupplier.java index 5539d2b..4b71c96 100644 --- a/common/src/main/java/com/zegelin/cassandra/exporter/FactoriesSupplier.java +++ b/common/src/main/java/com/zegelin/cassandra/exporter/FactoriesSupplier.java @@ -115,6 +115,7 @@ public interface CollectorConstructor { private final boolean perThreadTimingEnabled; private final Set tableLabels; private final Set excludedKeyspaces; + private final boolean resolveIP; private final Map tableMetricScopeFilters; @@ -123,6 +124,7 @@ public FactoriesSupplier(final MetadataFactory metadataFactory, final HarvesterO this.perThreadTimingEnabled = options.perThreadTimingEnabled; this.tableLabels = options.tableLabels; this.excludedKeyspaces = options.excludedKeyspaces; + this.resolveIP = options.resolveIP; this.tableMetricScopeFilters = ImmutableMap.builder() .put(TableMetricScope.NODE, options.nodeMetricsFilter) @@ -532,7 +534,7 @@ public List get() { final ImmutableList.Builder builder = ImmutableList.builder(); builder.add(FailureDetectorMBeanMetricFamilyCollector.factory(metadataFactory)); - builder.add(cache(StorageServiceMBeanMetricFamilyCollector.factory(metadataFactory, excludedKeyspaces), 5, TimeUnit.MINUTES)); + builder.add(cache(StorageServiceMBeanMetricFamilyCollector.factory(metadataFactory, excludedKeyspaces, resolveIP), 5, TimeUnit.MINUTES)); builder.add(MemoryPoolMXBeanMetricFamilyCollector.FACTORY); builder.add(GarbageCollectorMXBeanMetricFamilyCollector.FACTORY); diff --git a/common/src/main/java/com/zegelin/cassandra/exporter/Harvester.java b/common/src/main/java/com/zegelin/cassandra/exporter/Harvester.java index ae433c9..36b2092 100644 --- a/common/src/main/java/com/zegelin/cassandra/exporter/Harvester.java +++ b/common/src/main/java/com/zegelin/cassandra/exporter/Harvester.java @@ -131,6 +131,8 @@ public boolean equals(final Object o) { private final Set enabledGlobalLabels; private final boolean collectorTimingEnabled; + private final boolean resolveIP; + private final Map collectionTimes = new ConcurrentHashMap<>(); private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder() @@ -145,6 +147,7 @@ protected Harvester(final MetadataFactory metadataFactory, final HarvesterOption this.exclusions = options.exclusions; this.enabledGlobalLabels = options.globalLabels; this.collectorTimingEnabled = options.collectorTimingEnabled; + this.resolveIP = options.resolveIP; } protected void addCollectorFactory(final MBeanGroupMetricFamilyCollector.Factory factory) { diff --git a/common/src/main/java/com/zegelin/cassandra/exporter/cli/HarvesterOptions.java b/common/src/main/java/com/zegelin/cassandra/exporter/cli/HarvesterOptions.java index 580b0e4..55839dd 100644 --- a/common/src/main/java/com/zegelin/cassandra/exporter/cli/HarvesterOptions.java +++ b/common/src/main/java/com/zegelin/cassandra/exporter/cli/HarvesterOptions.java @@ -152,4 +152,8 @@ public void setExcludeSystemTables(final boolean excludeSystemTables) { excludedKeyspaces.addAll(CASSANDRA_SYSTEM_KEYSPACES); } + + @Option(names = "--enable-ip-resolve", + description = "Add label with hostname(s) from resolved IP(s)") + public boolean resolveIP; } diff --git a/common/src/main/java/com/zegelin/cassandra/exporter/collector/StorageServiceMBeanMetricFamilyCollector.java b/common/src/main/java/com/zegelin/cassandra/exporter/collector/StorageServiceMBeanMetricFamilyCollector.java index f7cc897..6a989c0 100644 --- a/common/src/main/java/com/zegelin/cassandra/exporter/collector/StorageServiceMBeanMetricFamilyCollector.java +++ b/common/src/main/java/com/zegelin/cassandra/exporter/collector/StorageServiceMBeanMetricFamilyCollector.java @@ -1,7 +1,9 @@ package com.zegelin.cassandra.exporter.collector; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; import com.zegelin.cassandra.exporter.MBeanGroupMetricFamilyCollector; import com.zegelin.cassandra.exporter.MetadataFactory; import com.zegelin.prometheus.domain.GaugeMetricFamily; @@ -14,12 +16,18 @@ import java.io.IOException; import java.net.InetAddress; +import java.net.UnknownHostException; import java.nio.file.FileStore; import java.nio.file.Files; import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; import java.util.Set; +import java.util.HashSet; +import java.util.StringTokenizer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; import java.util.stream.Stream; import static com.zegelin.cassandra.exporter.CassandraObjectNames.STORAGE_SERVICE_MBEAN_NAME; @@ -27,28 +35,31 @@ public class StorageServiceMBeanMetricFamilyCollector extends MBeanGroupMetricFamilyCollector { private static final Logger logger = LoggerFactory.getLogger(StorageServiceMBeanMetricFamilyCollector.class); - public static Factory factory(final MetadataFactory metadataFactory, final Set excludedKeyspaces) { + public static Factory factory(final MetadataFactory metadataFactory, final Set excludedKeyspaces, final boolean resolveIP) { return mBean -> { if (!STORAGE_SERVICE_MBEAN_NAME.apply(mBean.name)) return null; - return new StorageServiceMBeanMetricFamilyCollector((StorageServiceMBean) mBean.object, metadataFactory, excludedKeyspaces); + return new StorageServiceMBeanMetricFamilyCollector((StorageServiceMBean) mBean.object, metadataFactory, excludedKeyspaces, resolveIP); }; } private final StorageServiceMBean storageServiceMBean; private final MetadataFactory metadataFactory; private final Set excludedKeyspaces; - + private final boolean resolveIP; private final Map labeledFileStores; + private final Pattern tokenRangePattern = Pattern.compile("TokenRange\\(start_token:(-?\\d+), end_token:(-?\\d+)(, )?endpoints:\\[([^\\]]+)\\]"); + private StorageServiceMBeanMetricFamilyCollector(final StorageServiceMBean storageServiceMBean, - final MetadataFactory metadataFactory, final Set excludedKeyspaces) { + final MetadataFactory metadataFactory, final Set excludedKeyspaces, final boolean resolveIP) { this.storageServiceMBean = storageServiceMBean; this.metadataFactory = metadataFactory; this.excludedKeyspaces = excludedKeyspaces; + this.resolveIP = resolveIP; // determine the set of FileStores (i.e., mountpoints) for the Cassandra data/CL/cache directories // (which can be done once -- changing directories requires a server restart) @@ -76,6 +87,44 @@ private StorageServiceMBeanMetricFamilyCollector(final StorageServiceMBean stora this.labeledFileStores = ImmutableMap.copyOf(labeledFileStores); } + private Labels decodeTokenRange(String tokenRange, String localIP, String keyspace) { + HashMap m = Maps.newHashMap(metadataFactory.endpointLabels(localIP)); + HashSet endpoints = new HashSet(); + + m.put("keyspace", keyspace); + + // token range example: + // TokenRange(start_token:5585272669612250202, end_token:5664918566912044362, endpoints:[172.16.28.48, 172.16.28.166], rpc_endpoints:[172.16.28.48, 172.16.28.166], endpoint_details:[EndpointDetails(host:172.16.28.48, datacenter:eu-west_edge-irl1_profiles-bk, rack:1a), EndpointDetails(host:172.16.28.166, datacenter:eu-west_edge-irl1_profiles-bk, rack:1c)]) + // see https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/service/TokenRange.java + Matcher matcher = tokenRangePattern.matcher(tokenRange); + if (matcher.find()) { + m.put("start_token", matcher.group(1)); + m.put("end_token", matcher.group(2)); + + StringTokenizer st = new StringTokenizer(matcher.group(4), ","); + while (st.hasMoreTokens()) { + endpoints.add(st.nextToken().trim()); + } + + if (endpoints.remove(localIP)) { + m.put("neighbours_endpoints", String.join(", ", endpoints)); + if (resolveIP) + m.put("neighbours_hostnames", endpoints.stream().map(e -> { + try { + return InetAddress.getByName(e).getHostName(); + } catch (UnknownHostException ex) { + return e; + } + }).collect(Collectors.joining(", "))); + } else { + m.put("neighbours_endpoints", ""); + if (resolveIP) + m.put("neighbours_hostnames", ""); + } + } + return new Labels(m); + } + @Override public Stream collect() { final Stream.Builder metricFamilyStreamBuilder = Stream.builder(); @@ -91,6 +140,23 @@ public Stream collect() { metricFamilyStreamBuilder.add(new GaugeMetricFamily("cassandra_token_ownership_ratio", null, ownershipMetricStream)); } + { + String localIP = storageServiceMBean.getHostIdToEndpoint().get(storageServiceMBean.getLocalHostId()); + final Stream ownershipMetricStream = metadataFactory.keyspaces().stream() + .filter(keyspace -> !excludedKeyspaces.contains(keyspace)) + .flatMap(keyspace -> { + try { + return storageServiceMBean.describeRingJMX(keyspace).stream() + .map(e -> decodeTokenRange(e, localIP, keyspace)) + .filter(e -> !Strings.isNullOrEmpty(e.get("neighbours_endpoints"))) + .map(e -> new NumericMetric(e, 1.0f)); + } catch (IOException e) { + return Stream.empty(); + } + }); + metricFamilyStreamBuilder.add(new GaugeMetricFamily("cassandra_neighbours", null, ownershipMetricStream)); + } + { final Stream ownershipMetricStream = metadataFactory.keyspaces().stream() .filter(keyspace -> !excludedKeyspaces.contains(keyspace))