diff --git a/instrumentation/jmx-metrics/README.md b/instrumentation/jmx-metrics/README.md index 6aa194a8ad0e..96a2ae1f06fa 100644 --- a/instrumentation/jmx-metrics/README.md +++ b/instrumentation/jmx-metrics/README.md @@ -31,7 +31,7 @@ No targets are enabled by default. The supported target environments are listed - [kafka-broker](javaagent/kafka-broker.md) - [tomcat](library/tomcat.md) - [wildfly](library/wildfly.md) -- [hadoop](javaagent/hadoop.md) +- [hadoop](library/hadoop.md) The [jvm](library/jvm.md) metrics definitions are also included in the [jmx-metrics library](./library) to allow reusing them without instrumentation. When using instrumentation, the [runtime-telemetry](../runtime-telemetry) diff --git a/instrumentation/jmx-metrics/javaagent/hadoop.md b/instrumentation/jmx-metrics/javaagent/hadoop.md deleted file mode 100644 index 24e2b7cf78be..000000000000 --- a/instrumentation/jmx-metrics/javaagent/hadoop.md +++ /dev/null @@ -1,15 +0,0 @@ -# Hadoop Metrics - -Here is the list of metrics based on MBeans exposed by Hadoop. - -| Metric Name | Type | Attributes | Description | -| --------------------------------- | ------------- | ---------------- | ----------------------------------------------------- | -| hadoop.capacity.CapacityUsed | UpDownCounter | node_name | Current used capacity across all data nodes | -| hadoop.capacity.CapacityTotal | UpDownCounter | node_name | Current raw capacity of data nodes | -| hadoop.block.BlocksTotal | UpDownCounter | node_name | Current number of allocated blocks in the system | -| hadoop.block.MissingBlocks | UpDownCounter | node_name | Current number of missing blocks | -| hadoop.block.CorruptBlocks | UpDownCounter | node_name | Current number of blocks with corrupt replicas | -| hadoop.volume.VolumeFailuresTotal | UpDownCounter | node_name | Total number of volume failures across all data nodes | -| hadoop.file.FilesTotal | UpDownCounter | node_name | Current number of files and directories | -| hadoop.file.TotalLoad | UpDownCounter | node_name | Current number of connection | -| hadoop.datanode.Count | UpDownCounter | node_name, state | The Number of data nodes | diff --git a/instrumentation/jmx-metrics/javaagent/src/main/resources/jmx/rules/hadoop.yaml b/instrumentation/jmx-metrics/javaagent/src/main/resources/jmx/rules/hadoop.yaml deleted file mode 100644 index 82c32bd95ebb..000000000000 --- a/instrumentation/jmx-metrics/javaagent/src/main/resources/jmx/rules/hadoop.yaml +++ /dev/null @@ -1,63 +0,0 @@ ---- -rules: - - bean: Hadoop:service=NameNode,name=FSNamesystem - unit: "1" - prefix: hadoop. - metricAttribute: - node_name: param(tag.Hostname) - mapping: - CapacityUsed: - metric: capacity.CapacityUsed - type: updowncounter - unit: By - desc: Current used capacity across all data nodes - CapacityTotal: - metric: capacity.CapacityTotal - type: updowncounter - unit: By - BlocksTotal: - metric: block.BlocksTotal - type: updowncounter - unit: "{blocks}" - desc: Current number of allocated blocks in the system - MissingBlocks: - metric: block.MissingBlocks - type: updowncounter - unit: "{blocks}" - desc: Current number of missing blocks - CorruptBlocks: - metric: block.CorruptBlocks - type: updowncounter - unit: "{blocks}" - desc: Current number of blocks with corrupt replicas - VolumeFailuresTotal: - metric: volume.VolumeFailuresTotal - type: updowncounter - unit: "{volumes}" - desc: Total number of volume failures across all data nodes - metricAttribute: - direction: const(sent) - FilesTotal: - metric: file.FilesTotal - type: updowncounter - unit: "{files}" - desc: Current number of files and directories - TotalLoad: - metric: file.TotalLoad - type: updowncounter - unit: "{operations}" - desc: Current number of connections - NumLiveDataNodes: - metric: datenode.Count - type: updowncounter - unit: "{nodes}" - desc: The Number of data nodes - metricAttribute: - state: const(live) - NumDeadDataNodes: - metric: datenode.Count - type: updowncounter - unit: "{nodes}" - desc: The Number of data nodes - metricAttribute: - state: const(dead) diff --git a/instrumentation/jmx-metrics/javaagent/src/test/java/io/opentelemetry/instrumentation/javaagent/jmx/JmxMetricInsightInstallerTest.java b/instrumentation/jmx-metrics/javaagent/src/test/java/io/opentelemetry/instrumentation/javaagent/jmx/JmxMetricInsightInstallerTest.java index 50c029cf8b79..2f486ade2e54 100644 --- a/instrumentation/jmx-metrics/javaagent/src/test/java/io/opentelemetry/instrumentation/javaagent/jmx/JmxMetricInsightInstallerTest.java +++ b/instrumentation/jmx-metrics/javaagent/src/test/java/io/opentelemetry/instrumentation/javaagent/jmx/JmxMetricInsightInstallerTest.java @@ -31,8 +31,7 @@ class JmxMetricInsightInstallerTest { private static final String PATH_TO_ALL_EXISTING_RULES = "src/main/resources/jmx/rules"; private static final Set FILES_TO_BE_TESTED = - new HashSet<>( - Arrays.asList("activemq.yaml", "camel.yaml", "hadoop.yaml", "kafka-broker.yaml")); + new HashSet<>(Arrays.asList("activemq.yaml", "camel.yaml", "kafka-broker.yaml")); @Test void testToVerifyExistingRulesAreValid() throws Exception { diff --git a/instrumentation/jmx-metrics/library/hadoop.md b/instrumentation/jmx-metrics/library/hadoop.md new file mode 100644 index 000000000000..5a6bcbfd32ce --- /dev/null +++ b/instrumentation/jmx-metrics/library/hadoop.md @@ -0,0 +1,15 @@ +# Hadoop Metrics + +Here is the list of metrics based on MBeans exposed by Hadoop. + +| Metric Name | Type | Attributes | Description | +|---------------------------------|---------------|-------------------------------------|--------------------------------------------------------| +| hadoop.dfs.capacity | UpDownCounter | hadoop.node.name | Current raw capacity of data nodes. | +| hadoop.dfs.capacity.used | UpDownCounter | hadoop.node.name | Current used capacity across all data nodes. | +| hadoop.dfs.block.count | UpDownCounter | hadoop.node.name | Current number of allocated blocks in the system. | +| hadoop.dfs.block.missing | UpDownCounter | hadoop.node.name | Current number of missing blocks. | +| hadoop.dfs.block.corrupt | UpDownCounter | hadoop.node.name | Current number of blocks with corrupt replicas. | +| hadoop.dfs.volume.failure.count | Counter | hadoop.node.name | Total number of volume failures across all data nodes. | +| hadoop.dfs.file.count | UpDownCounter | hadoop.node.name | Current number of files and directories. | +| hadoop.dfs.connection.count | UpDownCounter | hadoop.node.name | Current number of connection. | +| hadoop.dfs.data_node.count | UpDownCounter | hadoop.node.name, hadoop.node.state | The number of data nodes. | diff --git a/instrumentation/jmx-metrics/library/src/main/resources/jmx/rules/hadoop.yaml b/instrumentation/jmx-metrics/library/src/main/resources/jmx/rules/hadoop.yaml new file mode 100644 index 000000000000..ca6173a8423a --- /dev/null +++ b/instrumentation/jmx-metrics/library/src/main/resources/jmx/rules/hadoop.yaml @@ -0,0 +1,71 @@ +--- +rules: + - bean: Hadoop:service=NameNode,name=FSNamesystem + prefix: hadoop.dfs. + metricAttribute: + hadoop.node.name: beanattr(tag\.Hostname) + mapping: + # hadoop.dfs.capacity + CapacityTotal: + metric: capacity + type: updowncounter + unit: By + desc: Current raw capacity of DataNodes. + # hadoop.dfs.capacity.used + CapacityUsed: + metric: capacity.used + type: updowncounter + unit: By + desc: Current used capacity across all DataNodes. + # hadoop.dfs.block.count + BlocksTotal: + metric: block.count + type: updowncounter + unit: "{block}" + desc: Current number of allocated blocks in the system. + # hadoop.dfs.block.missing + MissingBlocks: + metric: block.missing + type: updowncounter + unit: "{block}" + desc: Current number of missing blocks. + # hadoop.dfs.block.corrupt + CorruptBlocks: + metric: block.corrupt + type: updowncounter + unit: "{block}" + desc: Current number of blocks with corrupt replicas. + # hadoop.dfs.volume.failure.count + VolumeFailuresTotal: + metric: volume.failure.count + type: counter + unit: "{failure}" + desc: Total number of volume failures across all DataNodes. + # hadoop.dfs.file.count + FilesTotal: + metric: file.count + type: updowncounter + unit: "{file}" + desc: Current number of files and directories. + # hadoop.dfs.connection.count + TotalLoad: + metric: connection.count + type: updowncounter + unit: "{connection}" + desc: Current number of connections. + + # hadoop.dfs.data_node.count + NumLiveDataNodes: + metric: &metric data_node.count + type: &type updowncounter + unit: &unit "{node}" + desc: &desc The number of DataNodes. + metricAttribute: + hadoop.node.state: const(live) + NumDeadDataNodes: + metric: *metric + type: *type + unit: *unit + desc: *desc + metricAttribute: + hadoop.node.state: const(dead) diff --git a/instrumentation/jmx-metrics/library/src/test/java/io/opentelemetry/instrumentation/jmx/rules/HadoopTest.java b/instrumentation/jmx-metrics/library/src/test/java/io/opentelemetry/instrumentation/jmx/rules/HadoopTest.java new file mode 100644 index 000000000000..4bc0b7a1fdbc --- /dev/null +++ b/instrumentation/jmx-metrics/library/src/test/java/io/opentelemetry/instrumentation/jmx/rules/HadoopTest.java @@ -0,0 +1,177 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.instrumentation.jmx.rules; + +import static io.opentelemetry.instrumentation.jmx.rules.assertions.DataPointAttributes.attribute; +import static io.opentelemetry.instrumentation.jmx.rules.assertions.DataPointAttributes.attributeGroup; + +import io.opentelemetry.instrumentation.jmx.rules.assertions.AttributeMatcher; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.images.builder.Transferable; + +class HadoopTest extends TargetSystemTest { + + public static final String ENDPOINT_PLACEHOLDER = "<>"; + + @Test + void testMetrics_Hadoop2x() throws URISyntaxException, IOException { + List yamlFiles = Collections.singletonList("hadoop.yaml"); + + yamlFiles.forEach(this::validateYamlSyntax); + + // Hadoop startup script does not propagate env vars to launched hadoop daemons, + // so all the env vars needs to be embedded inside the hadoop-env.sh file + GenericContainer target = + new GenericContainer<>("bmedora/hadoop:2.9-base") + .withCopyToContainer( + Transferable.of(readAndPreprocessEnvFile("hadoop2-env.sh")), + "/hadoop/etc/hadoop/hadoop-env.sh") + .withCreateContainerCmdModifier(cmd -> cmd.withHostName("test-host")) + .withStartupTimeout(Duration.ofMinutes(3)) + .withExposedPorts(50070) + .waitingFor(Wait.forListeningPorts(50070)); + + copyAgentToTarget(target); + copyYamlFilesToTarget(target, yamlFiles); + + startTarget(target); + + verifyMetrics(createMetricsVerifier()); + } + + private String readAndPreprocessEnvFile(String fileName) throws URISyntaxException, IOException { + Path path = Paths.get(getClass().getClassLoader().getResource(fileName).toURI()); + + String data; + try (Stream lines = Files.lines(path)) { + data = + lines + .map(line -> line.replace(ENDPOINT_PLACEHOLDER, otlpEndpoint)) + .collect(Collectors.joining("\n")); + } + + return data; + } + + @Test + void testMetrics_Hadoop3x() throws URISyntaxException, IOException { + List yamlFiles = Collections.singletonList("hadoop.yaml"); + + yamlFiles.forEach(this::validateYamlSyntax); + + // Hadoop startup script does not propagate env vars to launched hadoop daemons, + // so all the env vars needs to be embedded inside the hadoop-env.sh file + GenericContainer target = + new GenericContainer<>("loum/hadoop-pseudo:3.3.6") + .withExposedPorts(9870, 9000) + .withCopyToContainer( + Transferable.of(readAndPreprocessEnvFile("hadoop3-env.sh")), + "/opt/hadoop/etc/hadoop/hadoop-env.sh") + .withCreateContainerCmdModifier(cmd -> cmd.withHostName("test-host")) + .waitingFor( + Wait.forListeningPorts(9870, 9000).withStartupTimeout(Duration.ofMinutes(3))); + + copyAgentToTarget(target); + copyYamlFilesToTarget(target, yamlFiles); + + startTarget(target); + + verifyMetrics(createMetricsVerifier()); + } + + private static MetricsVerifier createMetricsVerifier() { + AttributeMatcher nodeNameAttribute = attribute("hadoop.node.name", "test-host"); + + return MetricsVerifier.create() + .disableStrictMode() + .add( + "hadoop.dfs.capacity", + metric -> + metric + .hasDescription("Current raw capacity of DataNodes.") + .hasUnit("By") + .isUpDownCounter() + .hasDataPointsWithOneAttribute(nodeNameAttribute)) + .add( + "hadoop.dfs.capacity.used", + metric -> + metric + .hasDescription("Current used capacity across all DataNodes.") + .hasUnit("By") + .isUpDownCounter() + .hasDataPointsWithOneAttribute(nodeNameAttribute)) + .add( + "hadoop.dfs.block.count", + metric -> + metric + .hasDescription("Current number of allocated blocks in the system.") + .hasUnit("{block}") + .isUpDownCounter() + .hasDataPointsWithOneAttribute(nodeNameAttribute)) + .add( + "hadoop.dfs.block.missing", + metric -> + metric + .hasDescription("Current number of missing blocks.") + .hasUnit("{block}") + .isUpDownCounter() + .hasDataPointsWithOneAttribute(nodeNameAttribute)) + .add( + "hadoop.dfs.block.corrupt", + metric -> + metric + .hasDescription("Current number of blocks with corrupt replicas.") + .hasUnit("{block}") + .isUpDownCounter() + .hasDataPointsWithOneAttribute(nodeNameAttribute)) + .add( + "hadoop.dfs.volume.failure.count", + metric -> + metric + .hasDescription("Total number of volume failures across all DataNodes.") + .hasUnit("{failure}") + .isCounter() + .hasDataPointsWithOneAttribute(nodeNameAttribute)) + .add( + "hadoop.dfs.file.count", + metric -> + metric + .hasDescription("Current number of files and directories.") + .hasUnit("{file}") + .isUpDownCounter() + .hasDataPointsWithOneAttribute(nodeNameAttribute)) + .add( + "hadoop.dfs.connection.count", + metric -> + metric + .hasDescription("Current number of connections.") + .hasUnit("{connection}") + .isUpDownCounter() + .hasDataPointsWithOneAttribute(nodeNameAttribute)) + .add( + "hadoop.dfs.data_node.count", + metric -> + metric + .hasDescription("The number of DataNodes.") + .hasUnit("{node}") + .isUpDownCounter() + .hasDataPointsWithAttributes( + attributeGroup(attribute("hadoop.node.state", "live"), nodeNameAttribute), + attributeGroup(attribute("hadoop.node.state", "dead"), nodeNameAttribute))); + } +} diff --git a/instrumentation/jmx-metrics/library/src/test/java/io/opentelemetry/instrumentation/jmx/rules/TargetSystemTest.java b/instrumentation/jmx-metrics/library/src/test/java/io/opentelemetry/instrumentation/jmx/rules/TargetSystemTest.java index 4facd775f965..7deb553ebe52 100644 --- a/instrumentation/jmx-metrics/library/src/test/java/io/opentelemetry/instrumentation/jmx/rules/TargetSystemTest.java +++ b/instrumentation/jmx-metrics/library/src/test/java/io/opentelemetry/instrumentation/jmx/rules/TargetSystemTest.java @@ -63,7 +63,7 @@ public class TargetSystemTest { private static OtlpGrpcServer otlpServer; private static Path agentPath; private static Path testAppPath; - private static String otlpEndpoint; + protected static String otlpEndpoint; private GenericContainer targetSystem; private Collection> targetDependencies; @@ -150,7 +150,7 @@ protected static Map otelConfigProperties(List yamlFiles // disable runtime telemetry metrics config.put("otel.instrumentation.runtime-telemetry.enabled", "false"); // set yaml config files to test - config.put("otel.jmx.target", "tomcat"); + config.put("otel.jmx.target", "hadoop"); config.put( "otel.jmx.config", yamlFiles.stream() diff --git a/instrumentation/jmx-metrics/library/src/test/resources/hadoop2-env.sh b/instrumentation/jmx-metrics/library/src/test/resources/hadoop2-env.sh new file mode 100644 index 000000000000..aba009ffe153 --- /dev/null +++ b/instrumentation/jmx-metrics/library/src/test/resources/hadoop2-env.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +# Set Hadoop-specific environment variables here. + +# The only required environment variable is JAVA_HOME. All others are +# optional. When running a distributed configuration it is best to +# set JAVA_HOME in this file, so that it is correctly defined on +# remote nodes. + +# The java implementation to use. +export JAVA_HOME=${JAVA_HOME} + +# The jsvc implementation to use. Jsvc is required to run secure datanodes +# that bind to privileged ports to provide authentication of data transfer +# protocol. Jsvc is not required if SASL is configured for authentication of +# data transfer protocol using non-privileged ports. +#export JSVC_HOME=${JSVC_HOME} + +export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/etc/hadoop"} + +# Extra Java CLASSPATH elements. Automatically insert capacity-scheduler. +for f in "$HADOOP_HOME"/contrib/capacity-scheduler/*.jar; do + if [ "$HADOOP_CLASSPATH" ]; then + export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$f + else + export HADOOP_CLASSPATH=$f + fi +done + +# The maximum amount of heap to use, in MB. Default is 1000. +#export HADOOP_HEAPSIZE= +#export HADOOP_NAMENODE_INIT_HEAPSIZE="" + +# Enable extra debugging of Hadoop's JAAS binding, used to set up +# Kerberos security. +# export HADOOP_JAAS_DEBUG=true + +# Extra Java runtime options. Empty by default. +# For Kerberos debugging, an extended option set logs more invormation +# export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true -Dsun.security.krb5.debug=true -Dsun.security.spnego.debug" +export HADOOP_OPTS="$HADOOP_OPTS -Djava.net.preferIPv4Stack=true" + +# Command specific options appended to HADOOP_OPTS when specified +export HADOOP_NAMENODE_OPTS="$HADOOP_NAMENODE_OPTS -Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender}" + +export HADOOP_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS $HADOOP_DATANODE_OPTS" + +export HADOOP_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_SECONDARYNAMENODE_OPTS" + +export HADOOP_NFS3_OPTS="$HADOOP_NFS3_OPTS" +export HADOOP_PORTMAP_OPTS="-Xmx512m $HADOOP_PORTMAP_OPTS" + +# The following applies to multiple commands (fs, dfs, fsck, distcp etc) +export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS" +# set heap args when HADOOP_HEAPSIZE is empty +if [ "$HADOOP_HEAPSIZE" = "" ]; then + export HADOOP_CLIENT_OPTS="-Xmx512m $HADOOP_CLIENT_OPTS" +fi +#HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData $HADOOP_JAVA_PLATFORM_OPTS" + +# On secure datanodes, user to run the datanode as after dropping privileges. +# This **MUST** be uncommented to enable secure HDFS if using privileged ports +# to provide authentication of data transfer protocol. This **MUST NOT** be +# defined if SASL is configured for authentication of data transfer protocol +# using non-privileged ports. +export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER} + +# Where log files are stored. $HADOOP_HOME/logs by default. +#export HADOOP_LOG_DIR=${HADOOP_LOG_DIR}/$USER + +# Where log files are stored in the secure data environment. +#export HADOOP_SECURE_DN_LOG_DIR=${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER} + +### +# HDFS Mover specific parameters +### +# Specify the JVM options to be used when starting the HDFS Mover. +# These options will be appended to the options specified as HADOOP_OPTS +# and therefore may override any similar flags set in HADOOP_OPTS +# +# export HADOOP_MOVER_OPTS="" + +### +# Advanced Users Only! +### + +# The directory where pid files are stored. /tmp by default. +# NOTE: this should be set to a directory that can only be written to by +# the user that will run the hadoop daemons. Otherwise there is the +# potential for a symlink attack. +export HADOOP_PID_DIR=${HADOOP_PID_DIR} +export HADOOP_SECURE_DN_PID_DIR=${HADOOP_PID_DIR} + +# A string representing this instance of hadoop. $USER by default. +export HADOOP_IDENT_STRING=$USER + +# Java agent opts needed for +export JAVA_AGENT_OPTS="-javaagent:/opentelemetry-instrumentation-javaagent.jar" +export JAVA_AGENT_OPTS="$JAVA_AGENT_OPTS -Dotel.logs.exporter=none -Dotel.traces.exporter=none -Dotel.metrics.exporter=otlp" +export JAVA_AGENT_OPTS="$JAVA_AGENT_OPTS -Dotel.exporter.otlp.endpoint=<> -Dotel.exporter.otlp.protocol=grpc" +export JAVA_AGENT_OPTS="$JAVA_AGENT_OPTS -Dotel.metric.export.interval=5s -Dotel.instrumentation.runtime-telemetry.enabled=false" +export JAVA_AGENT_OPTS="$JAVA_AGENT_OPTS -Dotel.jmx.config=/hadoop.yaml" + +export HADOOP_NAMENODE_OPTS="$HADOOP_NAMENODE_OPTS $JAVA_AGENT_OPTS" diff --git a/instrumentation/jmx-metrics/library/src/test/resources/hadoop3-env.sh b/instrumentation/jmx-metrics/library/src/test/resources/hadoop3-env.sh new file mode 100644 index 000000000000..d206d2af7775 --- /dev/null +++ b/instrumentation/jmx-metrics/library/src/test/resources/hadoop3-env.sh @@ -0,0 +1,436 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Set Hadoop-specific environment variables here. + +## +## THIS FILE ACTS AS THE MASTER FILE FOR ALL HADOOP PROJECTS. +## SETTINGS HERE WILL BE READ BY ALL HADOOP COMMANDS. THEREFORE, +## ONE CAN USE THIS FILE TO SET YARN, HDFS, AND MAPREDUCE +## CONFIGURATION OPTIONS INSTEAD OF xxx-env.sh. +## +## Precedence rules: +## +## {yarn-env.sh|hdfs-env.sh} > hadoop2-env.sh > hard-coded defaults +## +## {YARN_xyz|HDFS_xyz} > HADOOP_xyz > hard-coded defaults +## + +# Many of the options here are built from the perspective that users +# may want to provide OVERWRITING values on the command line. +# For example: +# +# JAVA_HOME=/usr/java/testing hdfs dfs -ls +# +# Therefore, the vast majority (BUT NOT ALL!) of these defaults +# are configured for substitution and not append. If append +# is preferable, modify this file accordingly. + +### +# Generic settings for HADOOP +### + +# Technically, the only required environment variable is JAVA_HOME. +# All others are optional. However, the defaults are probably not +# preferred. Many sites configure these options outside of Hadoop, +# such as in /etc/profile.d + +# The java implementation to use. By default, this environment +# variable is REQUIRED on ALL platforms except OS X! +# export JAVA_HOME= +JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:bin/java::") +export JAVA_HOME + +# Location of Hadoop. By default, Hadoop will attempt to determine +# this location based upon its execution path. +# export HADOOP_HOME= + +# Location of Hadoop's configuration information. i.e., where this +# file is living. If this is not defined, Hadoop will attempt to +# locate it based upon its execution path. +# +# NOTE: It is recommend that this variable not be set here but in +# /etc/profile.d or equivalent. Some options (such as +# --config) may react strangely otherwise. +# +# export HADOOP_CONF_DIR=${HADOOP_HOME}/etc/hadoop + +# The maximum amount of heap to use (Java -Xmx). If no unit +# is provided, it will be converted to MB. Daemons will +# prefer any Xmx setting in their respective _OPT variable. +# There is no default; the JVM will autoscale based upon machine +# memory size. +# export HADOOP_HEAPSIZE_MAX= + +# The minimum amount of heap to use (Java -Xms). If no unit +# is provided, it will be converted to MB. Daemons will +# prefer any Xms setting in their respective _OPT variable. +# There is no default; the JVM will autoscale based upon machine +# memory size. +# export HADOOP_HEAPSIZE_MIN= + +# Enable extra debugging of Hadoop's JAAS binding, used to set up +# Kerberos security. +# export HADOOP_JAAS_DEBUG=true + +# Extra Java runtime options for all Hadoop commands. We don't support +# IPv6 yet/still, so by default the preference is set to IPv4. +# export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true" +# For Kerberos debugging, an extended option set logs more information +# export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true -Dsun.security.krb5.debug=true -Dsun.security.spnego.debug" + +# Some parts of the shell code may do special things dependent upon +# the operating system. We have to set this here. See the next +# section as to why.... +export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)} + +# Extra Java runtime options for some Hadoop commands +# and clients (i.e., hdfs dfs -blah). These get appended to HADOOP_OPTS for +# such commands. In most cases, # this should be left empty and +# let users supply it on the command line. +# export HADOOP_CLIENT_OPTS="" + +# +# A note about classpaths. +# +# By default, Apache Hadoop overrides Java's CLASSPATH +# environment variable. It is configured such +# that it starts out blank with new entries added after passing +# a series of checks (file/dir exists, not already listed aka +# de-deduplication). During de-deduplication, wildcards and/or +# directories are *NOT* expanded to keep it simple. Therefore, +# if the computed classpath has two specific mentions of +# awesome-methods-1.0.jar, only the first one added will be seen. +# If two directories are in the classpath that both contain +# awesome-methods-1.0.jar, then Java will pick up both versions. + +# An additional, custom CLASSPATH. Site-wide configs should be +# handled via the shellprofile functionality, utilizing the +# hadoop_add_classpath function for greater control and much +# harder for apps/end-users to accidentally override. +# Similarly, end users should utilize ${HOME}/.hadooprc . +# This variable should ideally only be used as a short-cut, +# interactive way for temporary additions on the command line. +# export HADOOP_CLASSPATH="/some/cool/path/on/your/machine" + +# Should HADOOP_CLASSPATH be first in the official CLASSPATH? +# export HADOOP_USER_CLASSPATH_FIRST="yes" + +# If HADOOP_USE_CLIENT_CLASSLOADER is set, the classpath along +# with the main jar are handled by a separate isolated +# client classloader when 'hadoop jar', 'yarn jar', or 'mapred job' +# is utilized. If it is set, HADOOP_CLASSPATH and +# HADOOP_USER_CLASSPATH_FIRST are ignored. +# export HADOOP_USE_CLIENT_CLASSLOADER=true + +# HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES overrides the default definition of +# system classes for the client classloader when HADOOP_USE_CLIENT_CLASSLOADER +# is enabled. Names ending in '.' (period) are treated as package names, and +# names starting with a '-' are treated as negative matches. For example, +# export HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES="-org.apache.hadoop.UserClass,java.,javax.,org.apache.hadoop." + +# Enable optional, bundled Hadoop features +# This is a comma delimited list. It may NOT be overridden via .hadooprc +# Entries may be added/removed as needed. +# export HADOOP_OPTIONAL_TOOLS="hadoop-aliyun,hadoop-openstack,hadoop-azure,hadoop-azure-datalake,hadoop-aws,hadoop-kafka" + +### +# Options for remote shell connectivity +### + +# There are some optional components of hadoop that allow for +# command and control of remote hosts. For example, +# start-dfs.sh will attempt to bring up all NNs, DNS, etc. + +# Options to pass to SSH when one of the "log into a host and +# start/stop daemons" scripts is executed +# export HADOOP_SSH_OPTS="-o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=10s" +export HADOOP_SSH_OPTS="-o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=10s -p 2222" + +# The built-in ssh handler will limit itself to 10 simultaneous connections. +# For pdsh users, this sets the fanout size ( -f ) +# Change this to increase/decrease as necessary. +# export HADOOP_SSH_PARALLEL=10 + +# Filename which contains all of the hosts for any remote execution +# helper scripts # such as workers.sh, start-dfs.sh, etc. +# export HADOOP_WORKERS="${HADOOP_CONF_DIR}/workers" + +### +# Options for all daemons +### +# + +# +# Many options may also be specified as Java properties. It is +# very common, and in many cases, desirable, to hard-set these +# in daemon _OPTS variables. Where applicable, the appropriate +# Java property is also identified. Note that many are re-used +# or set differently in certain contexts (e.g., secure vs +# non-secure) +# + +# Where (primarily) daemon log files are stored. +# ${HADOOP_HOME}/logs by default. +# Java property: hadoop.log.dir +# export HADOOP_LOG_DIR=${HADOOP_HOME}/logs +export HADOOP_LOG_DIR=/var/log/hadoop + +# A string representing this instance of hadoop. $USER by default. +# This is used in writing log and pid files, so keep that in mind! +# Java property: hadoop.id.str +# export HADOOP_IDENT_STRING=$USER + +# How many seconds to pause after stopping a daemon +# export HADOOP_STOP_TIMEOUT=5 + +# Where pid files are stored. /tmp by default. +# export HADOOP_PID_DIR=/tmp + +# Default log4j setting for interactive commands +# Java property: hadoop.root.logger +# export HADOOP_ROOT_LOGGER=INFO,console + +# Default log4j setting for daemons spawned explicitly by +# --daemon option of hadoop, hdfs, mapred and yarn command. +# Java property: hadoop.root.logger +# export HADOOP_DAEMON_ROOT_LOGGER=INFO,RFA + +# Default log level and output location for security-related messages. +# You will almost certainly want to change this on a per-daemon basis via +# the Java property (i.e., -Dhadoop.security.logger=foo). (Note that the +# defaults for the NN and 2NN override this by default.) +# Java property: hadoop.security.logger +# export HADOOP_SECURITY_LOGGER=INFO,NullAppender + +# Default process priority level +# Note that sub-processes will also run at this level! +# export HADOOP_NICENESS=0 + +# Default name for the service level authorization file +# Java property: hadoop.policy.file +# export HADOOP_POLICYFILE="hadoop-policy.xml" + +# +# NOTE: this is not used by default! <----- +# You can define variables right here and then re-use them later on. +# For example, it is common to use the same garbage collection settings +# for all the daemons. So one could define: +# +# export HADOOP_GC_SETTINGS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps" +# +# .. and then use it as per the b option under the namenode. + +### +# Secure/privileged execution +### + +# +# Out of the box, Hadoop uses jsvc from Apache Commons to launch daemons +# on privileged ports. This functionality can be replaced by providing +# custom functions. See hadoop-functions.sh for more information. +# + +# The jsvc implementation to use. Jsvc is required to run secure datanodes +# that bind to privileged ports to provide authentication of data transfer +# protocol. Jsvc is not required if SASL is configured for authentication of +# data transfer protocol using non-privileged ports. +# export JSVC_HOME=/usr/bin + +# +# This directory contains pids for secure and privileged processes. +#export HADOOP_SECURE_PID_DIR=${HADOOP_PID_DIR} + +# +# This directory contains the logs for secure and privileged processes. +# Java property: hadoop.log.dir +# export HADOOP_SECURE_LOG=${HADOOP_LOG_DIR} + +# +# When running a secure daemon, the default value of HADOOP_IDENT_STRING +# ends up being a bit bogus. Therefore, by default, the code will +# replace HADOOP_IDENT_STRING with HADOOP_xx_SECURE_USER. If one wants +# to keep HADOOP_IDENT_STRING untouched, then uncomment this line. +# export HADOOP_SECURE_IDENT_PRESERVE="true" + +### +# NameNode specific parameters +### + +# Default log level and output location for file system related change +# messages. For non-namenode daemons, the Java property must be set in +# the appropriate _OPTS if one wants something other than INFO,NullAppender +# Java property: hdfs.audit.logger +# export HDFS_AUDIT_LOGGER=INFO,NullAppender + +# Specify the JVM options to be used when starting the NameNode. +# These options will be appended to the options specified as HADOOP_OPTS +# and therefore may override any similar flags set in HADOOP_OPTS +# +# a) Set JMX options +# export HDFS_NAMENODE_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=1026" +# +# b) Set garbage collection logs +# export HDFS_NAMENODE_OPTS="${HADOOP_GC_SETTINGS} -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')" +# +# c) ... or set them directly +# export HDFS_NAMENODE_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')" + +# this is the default: +# export HDFS_NAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS" + +### +# SecondaryNameNode specific parameters +### +# Specify the JVM options to be used when starting the SecondaryNameNode. +# These options will be appended to the options specified as HADOOP_OPTS +# and therefore may override any similar flags set in HADOOP_OPTS +# +# This is the default: +# export HDFS_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS" + +### +# DataNode specific parameters +### +# Specify the JVM options to be used when starting the DataNode. +# These options will be appended to the options specified as HADOOP_OPTS +# and therefore may override any similar flags set in HADOOP_OPTS +# +# This is the default: +# export HDFS_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS" + +# On secure datanodes, user to run the datanode as after dropping privileges. +# This **MUST** be uncommented to enable secure HDFS if using privileged ports +# to provide authentication of data transfer protocol. This **MUST NOT** be +# defined if SASL is configured for authentication of data transfer protocol +# using non-privileged ports. +# This will replace the hadoop.id.str Java property in secure mode. +# export HDFS_DATANODE_SECURE_USER=hdfs + +# Supplemental options for secure datanodes +# By default, Hadoop uses jsvc which needs to know to launch a +# server jvm. +# export HDFS_DATANODE_SECURE_EXTRA_OPTS="-jvm server" + +### +# NFS3 Gateway specific parameters +### +# Specify the JVM options to be used when starting the NFS3 Gateway. +# These options will be appended to the options specified as HADOOP_OPTS +# and therefore may override any similar flags set in HADOOP_OPTS +# +# export HDFS_NFS3_OPTS="" + +# Specify the JVM options to be used when starting the Hadoop portmapper. +# These options will be appended to the options specified as HADOOP_OPTS +# and therefore may override any similar flags set in HADOOP_OPTS +# +# export HDFS_PORTMAP_OPTS="-Xmx512m" + +# Supplemental options for privileged gateways +# By default, Hadoop uses jsvc which needs to know to launch a +# server jvm. +# export HDFS_NFS3_SECURE_EXTRA_OPTS="-jvm server" + +# On privileged gateways, user to run the gateway as after dropping privileges +# This will replace the hadoop.id.str Java property in secure mode. +# export HDFS_NFS3_SECURE_USER=nfsserver + +### +# ZKFailoverController specific parameters +### +# Specify the JVM options to be used when starting the ZKFailoverController. +# These options will be appended to the options specified as HADOOP_OPTS +# and therefore may override any similar flags set in HADOOP_OPTS +# +# export HDFS_ZKFC_OPTS="" + +### +# QuorumJournalNode specific parameters +### +# Specify the JVM options to be used when starting the QuorumJournalNode. +# These options will be appended to the options specified as HADOOP_OPTS +# and therefore may override any similar flags set in HADOOP_OPTS +# +# export HDFS_JOURNALNODE_OPTS="" + +### +# HDFS Balancer specific parameters +### +# Specify the JVM options to be used when starting the HDFS Balancer. +# These options will be appended to the options specified as HADOOP_OPTS +# and therefore may override any similar flags set in HADOOP_OPTS +# +# export HDFS_BALANCER_OPTS="" + +### +# HDFS Mover specific parameters +### +# Specify the JVM options to be used when starting the HDFS Mover. +# These options will be appended to the options specified as HADOOP_OPTS +# and therefore may override any similar flags set in HADOOP_OPTS +# +# export HDFS_MOVER_OPTS="" + +### +# Router-based HDFS Federation specific parameters +# Specify the JVM options to be used when starting the RBF Routers. +# These options will be appended to the options specified as HADOOP_OPTS +# and therefore may override any similar flags set in HADOOP_OPTS +# +# export HDFS_DFSROUTER_OPTS="" + +### +# HDFS StorageContainerManager specific parameters +### +# Specify the JVM options to be used when starting the HDFS Storage Container Manager. +# These options will be appended to the options specified as HADOOP_OPTS +# and therefore may override any similar flags set in HADOOP_OPTS +# +# export HDFS_STORAGECONTAINERMANAGER_OPTS="" + +### +# Advanced Users Only! +### + +# +# When building Hadoop, one can add the class paths to the commands +# via this special env var: +# export HADOOP_ENABLE_BUILD_PATHS="true" + +# +# To prevent accidents, shell commands be (superficially) locked +# to only allow certain users to execute certain subcommands. +# It uses the format of (command)_(subcommand)_USER. +# +# For example, to limit who can execute the namenode command, +# export HDFS_NAMENODE_USER=hdfs +export HDFS_NAMENODE_USER=hdfs +export HDFS_DATANODE_USER=hdfs +export HDFS_SECONDARYNAMENODE_USER=hdfs +export YARN_RESOURCEMANAGER_USER=hdfs +export YARN_NODEMANAGER_USER=hdfs + +# Java agent opts needed for +export JAVA_AGENT_OPTS="-javaagent:/opentelemetry-instrumentation-javaagent.jar" +export JAVA_AGENT_OPTS="$JAVA_AGENT_OPTS -Dotel.logs.exporter=none -Dotel.traces.exporter=none -Dotel.metrics.exporter=otlp" +export JAVA_AGENT_OPTS="$JAVA_AGENT_OPTS -Dotel.exporter.otlp.endpoint=<> -Dotel.exporter.otlp.protocol=grpc" +export JAVA_AGENT_OPTS="$JAVA_AGENT_OPTS -Dotel.metric.export.interval=5s -Dotel.instrumentation.runtime-telemetry.enabled=false" +export JAVA_AGENT_OPTS="$JAVA_AGENT_OPTS -Dotel.jmx.config=/hadoop.yaml" + +export HDFS_NAMENODE_OPTS="$HDFS_NAMENODE_OPTS $JAVA_AGENT_OPTS"