From 38c3dad51a2f9a197c99e24ba57731fb5f506551 Mon Sep 17 00:00:00 2001 From: Anurag Goyal Date: Wed, 10 Jun 2026 23:17:43 +0530 Subject: [PATCH 1/4] regenerate topology mappings on datanode install --- .../HDFS/package/scripts/hdfs_namenode.py | 10 ++++++++ .../HDFS/package/scripts/params_linux.py | 7 ++++++ .../templates/topology_mappings.data.j2 | 24 +++++++++++++++++++ 3 files changed, 41 insertions(+) create mode 100644 ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/templates/topology_mappings.data.j2 diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/hdfs_namenode.py index 5dc11b4c7e9..a7ef353e705 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/hdfs_namenode.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/hdfs_namenode.py @@ -570,6 +570,16 @@ def decommission(): ) pass + # regenerate topology_mappings.data + File( + params.net_topology_mapping_data_file_path, + content=Template("topology_mappings.data.j2"), + owner=params.hdfs_user, + group=params.user_group, + mode=0o644, + only_if=format("test -d {hadoop_conf_dir}") + ) + if not params.update_files_only: Execute(nn_kinit_cmd, user=hdfs_user) diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/params_linux.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/params_linux.py index be6e21b44fe..a18ad0abdb0 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/params_linux.py @@ -236,7 +236,14 @@ kinit_path_local = get_kinit_path( default("/configurations/kerberos-env/executable_search_paths", None) ) + +# topology files +net_topology_mapping_data_file_path = os.path.join(hadoop_conf_dir, "topology_mappings.data") + # hosts +all_hosts = default("/clusterHostInfo/all_hosts", []) +all_racks = default("/clusterHostInfo/all_racks", []) +all_ipv4_ips = default("/clusterHostInfo/all_ipv4_ips", []) hostname = config["agentLevelParams"]["hostname"] rm_host = default("/clusterHostInfo/resourcemanager_hosts", []) public_hostname = config["agentLevelParams"]["public_hostname"] diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/templates/topology_mappings.data.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/templates/topology_mappings.data.j2 new file mode 100644 index 00000000000..15034d63063 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/templates/topology_mappings.data.j2 @@ -0,0 +1,24 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 + # +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} +[network_topology] +{% for host in all_hosts %} +{% if host in slave_hosts %} +{{host}}={{all_racks[loop.index-1]}} +{{all_ipv4_ips[loop.index-1]}}={{all_racks[loop.index-1]}} +{% endif %} +{% endfor %} From d1c63691fe7ee6ff16a24e76041d9015bd12f7c4 Mon Sep 17 00:00:00 2001 From: Anurag Goyal Date: Wed, 10 Jun 2026 23:17:56 +0530 Subject: [PATCH 2/4] regenerate topology mappings on nodemanager install --- .../YARN/package/scripts/params_linux.py | 9 +++++++ .../YARN/package/scripts/resourcemanager.py | 10 ++++++++ .../templates/topology_mappings.data.j2 | 24 +++++++++++++++++++ 3 files changed, 43 insertions(+) create mode 100644 ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/topology_mappings.data.j2 diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/params_linux.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/params_linux.py index 709e95ba4f1..08ac611b5fb 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/params_linux.py @@ -377,6 +377,15 @@ def get_spark_version(service_name, component_name, yarn_version): user_group = config["configurations"]["cluster-env"]["user_group"] +# topology files +net_topology_mapping_data_file_path = os.path.join(hadoop_conf_dir, "topology_mappings.data") + +# hosts +all_hosts = default("/clusterHostInfo/all_hosts", []) +all_racks = default("/clusterHostInfo/all_racks", []) +all_ipv4_ips = default("/clusterHostInfo/all_ipv4_ips", []) +slave_hosts = default("/clusterHostInfo/datanode_hosts", []) + # exclude file if "all_decommissioned_hosts" in config["commandParams"]: exclude_hosts = config["commandParams"]["all_decommissioned_hosts"].split(",") diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/resourcemanager.py index 6b7ab4b4b35..c17f9ac5525 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/resourcemanager.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/resourcemanager.py @@ -188,6 +188,16 @@ def decommission(self, env): group=user_group, ) + # regenerate topology_mappings.data + File( + params.net_topology_mapping_data_file_path, + content=Template("topology_mappings.data.j2"), + owner=params.hdfs_user, + group=params.user_group, + mode=0o644, + only_if=format("test -d {hadoop_conf_dir}") + ) + if params.update_files_only == False: Execute( yarn_refresh_cmd, environment={"PATH": params.execute_path}, user=yarn_user diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/topology_mappings.data.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/topology_mappings.data.j2 new file mode 100644 index 00000000000..15034d63063 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/topology_mappings.data.j2 @@ -0,0 +1,24 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 + # +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} +[network_topology] +{% for host in all_hosts %} +{% if host in slave_hosts %} +{{host}}={{all_racks[loop.index-1]}} +{{all_ipv4_ips[loop.index-1]}}={{all_racks[loop.index-1]}} +{% endif %} +{% endfor %} From 2712f9bd84d8d1d786d9e7954684f783914d80e0 Mon Sep 17 00:00:00 2001 From: Anurag Goyal Date: Thu, 11 Jun 2026 00:15:45 +0530 Subject: [PATCH 3/4] include datanodes and nodemanagers --- .../main/resources/stack-hooks/before-START/scripts/params.py | 1 + .../before-START/templates/topology_mappings.data.j2 | 2 +- .../services/HDFS/package/templates/topology_mappings.data.j2 | 2 +- .../services/YARN/package/templates/topology_mappings.data.j2 | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ambari-server/src/main/resources/stack-hooks/before-START/scripts/params.py b/ambari-server/src/main/resources/stack-hooks/before-START/scripts/params.py index e2769e4e5cd..cef46737ba9 100644 --- a/ambari-server/src/main/resources/stack-hooks/before-START/scripts/params.py +++ b/ambari-server/src/main/resources/stack-hooks/before-START/scripts/params.py @@ -385,6 +385,7 @@ all_racks = default("/clusterHostInfo/all_racks", []) all_ipv4_ips = default("/clusterHostInfo/all_ipv4_ips", []) slave_hosts = default("/clusterHostInfo/datanode_hosts", []) +nm_hosts = default("/clusterHostInfo/nodemanager_hosts", []) # topology files net_topology_script_file_path = "/etc/hadoop/conf/topology_script.py" diff --git a/ambari-server/src/main/resources/stack-hooks/before-START/templates/topology_mappings.data.j2 b/ambari-server/src/main/resources/stack-hooks/before-START/templates/topology_mappings.data.j2 index 15034d63063..1e761534c62 100644 --- a/ambari-server/src/main/resources/stack-hooks/before-START/templates/topology_mappings.data.j2 +++ b/ambari-server/src/main/resources/stack-hooks/before-START/templates/topology_mappings.data.j2 @@ -17,7 +17,7 @@ #} [network_topology] {% for host in all_hosts %} -{% if host in slave_hosts %} +{% if host in slave_hosts or host in nm_hosts %} {{host}}={{all_racks[loop.index-1]}} {{all_ipv4_ips[loop.index-1]}}={{all_racks[loop.index-1]}} {% endif %} diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/templates/topology_mappings.data.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/templates/topology_mappings.data.j2 index 15034d63063..2f682da4e54 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/templates/topology_mappings.data.j2 +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/templates/topology_mappings.data.j2 @@ -17,7 +17,7 @@ #} [network_topology] {% for host in all_hosts %} -{% if host in slave_hosts %} +{% if host in slave_hosts or host in nm_host %} {{host}}={{all_racks[loop.index-1]}} {{all_ipv4_ips[loop.index-1]}}={{all_racks[loop.index-1]}} {% endif %} diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/topology_mappings.data.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/topology_mappings.data.j2 index 15034d63063..1e761534c62 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/topology_mappings.data.j2 +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/topology_mappings.data.j2 @@ -17,7 +17,7 @@ #} [network_topology] {% for host in all_hosts %} -{% if host in slave_hosts %} +{% if host in slave_hosts or host in nm_hosts %} {{host}}={{all_racks[loop.index-1]}} {{all_ipv4_ips[loop.index-1]}}={{all_racks[loop.index-1]}} {% endif %} From 03712e75528b9df1334060d88bfca3ee61ee2952 Mon Sep 17 00:00:00 2001 From: Anurag Goyal Date: Sun, 14 Jun 2026 20:16:37 +0530 Subject: [PATCH 4/4] fix indentation --- .../HDFS/package/scripts/hdfs_namenode.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/hdfs_namenode.py index a7ef353e705..fb524397e88 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/hdfs_namenode.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/hdfs_namenode.py @@ -570,15 +570,15 @@ def decommission(): ) pass - # regenerate topology_mappings.data - File( - params.net_topology_mapping_data_file_path, - content=Template("topology_mappings.data.j2"), - owner=params.hdfs_user, - group=params.user_group, - mode=0o644, - only_if=format("test -d {hadoop_conf_dir}") - ) + # regenerate topology_mappings.data + File( + params.net_topology_mapping_data_file_path, + content=Template("topology_mappings.data.j2"), + owner=params.hdfs_user, + group=params.user_group, + mode=0o644, + only_if=format("test -d {hadoop_conf_dir}") + ) if not params.update_files_only: Execute(nn_kinit_cmd, user=hdfs_user)