diff --git a/ambari-server/src/main/resources/stack-hooks/before-START/scripts/params.py b/ambari-server/src/main/resources/stack-hooks/before-START/scripts/params.py index e2769e4e5cd..cef46737ba9 100644 --- a/ambari-server/src/main/resources/stack-hooks/before-START/scripts/params.py +++ b/ambari-server/src/main/resources/stack-hooks/before-START/scripts/params.py @@ -385,6 +385,7 @@ all_racks = default("/clusterHostInfo/all_racks", []) all_ipv4_ips = default("/clusterHostInfo/all_ipv4_ips", []) slave_hosts = default("/clusterHostInfo/datanode_hosts", []) +nm_hosts = default("/clusterHostInfo/nodemanager_hosts", []) # topology files net_topology_script_file_path = "/etc/hadoop/conf/topology_script.py" diff --git a/ambari-server/src/main/resources/stack-hooks/before-START/templates/topology_mappings.data.j2 b/ambari-server/src/main/resources/stack-hooks/before-START/templates/topology_mappings.data.j2 index 15034d63063..1e761534c62 100644 --- a/ambari-server/src/main/resources/stack-hooks/before-START/templates/topology_mappings.data.j2 +++ b/ambari-server/src/main/resources/stack-hooks/before-START/templates/topology_mappings.data.j2 @@ -17,7 +17,7 @@ #} [network_topology] {% for host in all_hosts %} -{% if host in slave_hosts %} +{% if host in slave_hosts or host in nm_hosts %} {{host}}={{all_racks[loop.index-1]}} {{all_ipv4_ips[loop.index-1]}}={{all_racks[loop.index-1]}} {% endif %} diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/hdfs_namenode.py index 5dc11b4c7e9..fb524397e88 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/hdfs_namenode.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/hdfs_namenode.py @@ -570,6 +570,16 @@ def decommission(): ) pass + # regenerate topology_mappings.data + File( + params.net_topology_mapping_data_file_path, + content=Template("topology_mappings.data.j2"), + owner=params.hdfs_user, + group=params.user_group, + mode=0o644, + only_if=format("test -d {hadoop_conf_dir}") + ) + if not params.update_files_only: Execute(nn_kinit_cmd, user=hdfs_user) diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/params_linux.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/params_linux.py index be6e21b44fe..a18ad0abdb0 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/params_linux.py @@ -236,7 +236,14 @@ kinit_path_local = get_kinit_path( default("/configurations/kerberos-env/executable_search_paths", None) ) + +# topology files +net_topology_mapping_data_file_path = os.path.join(hadoop_conf_dir, "topology_mappings.data") + # hosts +all_hosts = default("/clusterHostInfo/all_hosts", []) +all_racks = default("/clusterHostInfo/all_racks", []) +all_ipv4_ips = default("/clusterHostInfo/all_ipv4_ips", []) hostname = config["agentLevelParams"]["hostname"] rm_host = default("/clusterHostInfo/resourcemanager_hosts", []) public_hostname = config["agentLevelParams"]["public_hostname"] diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/templates/topology_mappings.data.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/templates/topology_mappings.data.j2 new file mode 100644 index 00000000000..2f682da4e54 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/templates/topology_mappings.data.j2 @@ -0,0 +1,24 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 + # +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} +[network_topology] +{% for host in all_hosts %} +{% if host in slave_hosts or host in nm_host %} +{{host}}={{all_racks[loop.index-1]}} +{{all_ipv4_ips[loop.index-1]}}={{all_racks[loop.index-1]}} +{% endif %} +{% endfor %} diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/params_linux.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/params_linux.py index 709e95ba4f1..08ac611b5fb 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/params_linux.py @@ -377,6 +377,15 @@ def get_spark_version(service_name, component_name, yarn_version): user_group = config["configurations"]["cluster-env"]["user_group"] +# topology files +net_topology_mapping_data_file_path = os.path.join(hadoop_conf_dir, "topology_mappings.data") + +# hosts +all_hosts = default("/clusterHostInfo/all_hosts", []) +all_racks = default("/clusterHostInfo/all_racks", []) +all_ipv4_ips = default("/clusterHostInfo/all_ipv4_ips", []) +slave_hosts = default("/clusterHostInfo/datanode_hosts", []) + # exclude file if "all_decommissioned_hosts" in config["commandParams"]: exclude_hosts = config["commandParams"]["all_decommissioned_hosts"].split(",") diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/resourcemanager.py index 6b7ab4b4b35..c17f9ac5525 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/resourcemanager.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/resourcemanager.py @@ -188,6 +188,16 @@ def decommission(self, env): group=user_group, ) + # regenerate topology_mappings.data + File( + params.net_topology_mapping_data_file_path, + content=Template("topology_mappings.data.j2"), + owner=params.hdfs_user, + group=params.user_group, + mode=0o644, + only_if=format("test -d {hadoop_conf_dir}") + ) + if params.update_files_only == False: Execute( yarn_refresh_cmd, environment={"PATH": params.execute_path}, user=yarn_user diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/topology_mappings.data.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/topology_mappings.data.j2 new file mode 100644 index 00000000000..1e761534c62 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/topology_mappings.data.j2 @@ -0,0 +1,24 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 + # +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} +[network_topology] +{% for host in all_hosts %} +{% if host in slave_hosts or host in nm_hosts %} +{{host}}={{all_racks[loop.index-1]}} +{{all_ipv4_ips[loop.index-1]}}={{all_racks[loop.index-1]}} +{% endif %} +{% endfor %}