aws · hanwen-cluster · Jan 24, 2025 · Jan 24, 2025
@@ -0,0 +1,217 @@
+#!/bin/bash
+set -xe
+
+cd /etc/chef
+cat << 'EOF' > support_secondary_network_interface_patch
+Subject: [PATCH] Configure proper priority for route tables
+Raise priority of ParallelCluster configured route tables (#2857)
+Support attaching multiple network interfaces to the same network card
+---
+Index: cookbooks/aws-parallelcluster-environment/files/amazon-2023/network_interfaces/configure_nw_interface.sh
+<+>UTF-8
+===================================================================
+diff --git a/cookbooks/aws-parallelcluster-environment/files/amazon-2023/network_interfaces/configure_nw_interface.sh b/cookbooks/aws-parallelcluster-environment/files/amazon-2023/network_interfaces/configure_nw_interface.sh
+--- a/cookbooks/aws-parallelcluster-environment/files/amazon-2023/network_interfaces/configure_nw_interface.sh	(revision 9fe7e20db84ca556bcb830210c1f9ea852a38a89)
++++ b/cookbooks/aws-parallelcluster-environment/files/amazon-2023/network_interfaces/configure_nw_interface.sh	(revision e3949f39db1248a6a373283dc6132ac2edff1ff9)
+@@ -4,7 +4,8 @@
+
+ if
+   [ -z "${DEVICE_NAME}" ] ||          # name of the device
+-  [ -z "${DEVICE_NUMBER}" ] ||       # number of the device
++  [ -z "${DEVICE_NUMBER}" ] ||       # index of the device
++  [ -z "${NETWORK_CARD_INDEX}" ] ||   # index of the network card
+   [ -z "${DEVICE_IP_ADDRESS}" ] ||       # ip of the device
+   [ -z "${MAC}" ] ||                 # mac address of the device
+   [ -z "${CIDR_BLOCK}" ]                 # CIDR block of the subnet
+@@ -12,7 +13,7 @@
+   echo 'One or more environment variables missing'
+   exit 1
+ fi
+-echo "Configuring NIC, Device name: ${DEVICE_NAME}, Device number: ${DEVICE_NUMBER}"
++echo "Configuring NIC, Device name: ${DEVICE_NAME}, Device number: ${DEVICE_NUMBER}, Network card index:${NETWORK_CARD_INDEX}"
+
+ configuration_directory="/etc/systemd/network"
+ file_name="70-${DEVICE_NAME}.network"
+@@ -23,12 +24,13 @@
+
+ cd "$configuration_directory"
+
+-ROUTE_TABLE=100${DEVICE_NUMBER}
++SUFFIX=$NETWORK_CARD_INDEX$(printf "%02d" $DEVICE_NUMBER)
++ROUTE_TABLE="$(( $SUFFIX + 1000 ))"
+
+ ln -s /usr/lib/systemd/network/80-ec2.network ${file_name} # Use default EC2 configuration. This include MTU, etc.
+
+ /bin/cat <<EOF > ${sub_directory}/eni.conf
+-# Configuration for ${DEVICE_NUMBER} generated by ParallelCluster
++# Configuration for network card: ${NETWORK_CARD_INDEX}, device number: ${DEVICE_NUMBER} generated by ParallelCluster
+ # This is inspired by https://github.com/amazonlinux/amazon-ec2-net-utils/blob/v2.4.1/lib/lib.sh
+ [Match]
+ MACAddress=${MAC}
+Index: cookbooks/aws-parallelcluster-environment/files/default/network_interfaces/configure_nw_interface.sh
+IDEA additional info:
+Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
+<+>UTF-8
+===================================================================
+diff --git a/cookbooks/aws-parallelcluster-environment/files/default/network_interfaces/configure_nw_interface.sh b/cookbooks/aws-parallelcluster-environment/files/default/network_interfaces/configure_nw_interface.sh
+--- a/cookbooks/aws-parallelcluster-environment/files/default/network_interfaces/configure_nw_interface.sh	(revision 9fe7e20db84ca556bcb830210c1f9ea852a38a89)
++++ b/cookbooks/aws-parallelcluster-environment/files/default/network_interfaces/configure_nw_interface.sh	(revision e3949f39db1248a6a373283dc6132ac2edff1ff9)
+@@ -10,7 +10,8 @@
+
+ if
+   [ -z "${DEVICE_NAME}" ] ||          # name of the device
+-  [ -z "${DEVICE_NUMBER}" ] ||        # number of the device
++  [ -z "${DEVICE_NUMBER}" ] ||        # index of the device
++  [ -z "${NETWORK_CARD_INDEX}" ] ||   # index of the network card
+   [ -z "${GW_IP_ADDRESS}" ] ||        # gateway ip address
+   [ -z "${DEVICE_IP_ADDRESS}" ] ||    # ip address to assign to the interface
+   [ -z "${CIDR_PREFIX_LENGTH}" ] ||   # the prefix length of the device IP cidr block
+@@ -20,9 +21,11 @@
+   exit 1
+ fi
+
+-ROUTE_TABLE="100${DEVICE_NUMBER}"
++SUFFIX=$NETWORK_CARD_INDEX$(printf "%02d" $DEVICE_NUMBER)
+
+-echo "Configuring ${DEVICE_NAME} with IP:${DEVICE_IP_ADDRESS} CIDR_PREFIX:${CIDR_PREFIX_LENGTH} NETMASK:${NETMASK} GW:${GW_IP_ADDRESS} ROUTING_TABLE:${ROUTE_TABLE}"
++ROUTE_TABLE="$(( $SUFFIX + 1000 ))"
++
++echo "Configuring device name: ${DEVICE_NAME} with IP:${DEVICE_IP_ADDRESS} CIDR_PREFIX:${CIDR_PREFIX_LENGTH} NETMASK:${NETMASK} GW:${GW_IP_ADDRESS} ROUTING_TABLE:${ROUTE_TABLE}"
+
+ # config file
+ FILE="/etc/sysconfig/network-scripts/ifcfg-${DEVICE_NAME}"
+Index: cookbooks/aws-parallelcluster-environment/files/redhat-8.network_interfaces/configure_nw_interface.sh
+IDEA additional info:
+Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
+<+>UTF-8
+===================================================================
+diff --git a/cookbooks/aws-parallelcluster-environment/files/redhat-8.network_interfaces/configure_nw_interface.sh b/cookbooks/aws-parallelcluster-environment/files/redhat-8.network_interfaces/configure_nw_interface.sh
+--- a/cookbooks/aws-parallelcluster-environment/files/redhat-8.network_interfaces/configure_nw_interface.sh	(revision 9fe7e20db84ca556bcb830210c1f9ea852a38a89)
++++ b/cookbooks/aws-parallelcluster-environment/files/redhat-8.network_interfaces/configure_nw_interface.sh	(revision e3949f39db1248a6a373283dc6132ac2edff1ff9)
+@@ -13,7 +13,8 @@
+
+ if
+   [ -z "${DEVICE_NAME}" ] ||          # name of the device
+-  [ -z "${DEVICE_NUMBER}" ] ||        # number of the device
++  [ -z "${DEVICE_NUMBER}" ] ||        # index of the device
++  [ -z "${NETWORK_CARD_INDEX}" ] ||   # index of the network card
+   [ -z "${GW_IP_ADDRESS}" ] ||        # gateway ip address
+   [ -z "${DEVICE_IP_ADDRESS}" ] ||    # ip address to assign to the interface
+   [ -z "${CIDR_PREFIX_LENGTH}" ]      # the prefix length of the device IP cidr block
+@@ -23,9 +24,10 @@
+ fi
+
+ con_name="System ${DEVICE_NAME}"
+-route_table="100${DEVICE_NUMBER}"
+-priority="100${DEVICE_NUMBER}"
+-metric="100${DEVICE_NUMBER}"
++SUFFIX=$NETWORK_CARD_INDEX$(printf "%02d" $DEVICE_NUMBER)
++route_table="$(( $SUFFIX + 1000 ))"
++priority="${route_table}"
++metric="${route_table}"
+
+ # Rename connection
+ original_con_name=`nmcli -t -f GENERAL.CONNECTION device show ${DEVICE_NAME} | cut -f2 -d':'`
+Index: cookbooks/aws-parallelcluster-environment/files/rocky/network_interfaces/configure_nw_interface.sh
+IDEA additional info:
+Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
+<+>UTF-8
+===================================================================
+diff --git a/cookbooks/aws-parallelcluster-environment/files/rocky/network_interfaces/configure_nw_interface.sh b/cookbooks/aws-parallelcluster-environment/files/rocky/network_interfaces/configure_nw_interface.sh
+--- a/cookbooks/aws-parallelcluster-environment/files/rocky/network_interfaces/configure_nw_interface.sh	(revision 9fe7e20db84ca556bcb830210c1f9ea852a38a89)
++++ b/cookbooks/aws-parallelcluster-environment/files/rocky/network_interfaces/configure_nw_interface.sh	(revision e3949f39db1248a6a373283dc6132ac2edff1ff9)
+@@ -13,7 +13,8 @@
+
+ if
+   [ -z "${DEVICE_NAME}" ] ||          # name of the device
+-  [ -z "${DEVICE_NUMBER}" ] ||        # number of the device
++  [ -z "${DEVICE_NUMBER}" ] ||        # index of the device
++  [ -z "${NETWORK_CARD_INDEX}" ] ||   # index of the network card
+   [ -z "${GW_IP_ADDRESS}" ] ||        # gateway ip address
+   [ -z "${DEVICE_IP_ADDRESS}" ] ||    # ip address to assign to the interface
+   [ -z "${CIDR_PREFIX_LENGTH}" ]      # the prefix length of the device IP cidr block
+@@ -23,9 +24,10 @@
+ fi
+
+ con_name="System ${DEVICE_NAME}"
+-route_table="100${DEVICE_NUMBER}"
+-priority="100${DEVICE_NUMBER}"
+-metric="100${DEVICE_NUMBER}"
++SUFFIX=$NETWORK_CARD_INDEX$(printf "%02d" $DEVICE_NUMBER)
++route_table="$(( $SUFFIX + 1000 ))"
++priority="${route_table}"
++metric="${route_table}"
+
+ # Rename connection
+ original_con_name=`nmcli -t -f GENERAL.CONNECTION device show ${DEVICE_NAME} | cut -f2 -d':'`
+Index: cookbooks/aws-parallelcluster-environment/files/ubuntu/network_interfaces/configure_nw_interface.sh
+IDEA additional info:
+Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
+<+>UTF-8
+===================================================================
+diff --git a/cookbooks/aws-parallelcluster-environment/files/ubuntu/network_interfaces/configure_nw_interface.sh b/cookbooks/aws-parallelcluster-environment/files/ubuntu/network_interfaces/configure_nw_interface.sh
+--- a/cookbooks/aws-parallelcluster-environment/files/ubuntu/network_interfaces/configure_nw_interface.sh	(revision 9fe7e20db84ca556bcb830210c1f9ea852a38a89)
++++ b/cookbooks/aws-parallelcluster-environment/files/ubuntu/network_interfaces/configure_nw_interface.sh	(revision e3949f39db1248a6a373283dc6132ac2edff1ff9)
+@@ -10,7 +10,8 @@
+
+ if
+   [ -z "${DEVICE_NAME}" ] ||          # name of the device
+-  [ -z "${DEVICE_NUMBER}" ] ||        # number of the device
++  [ -z "${DEVICE_NUMBER}" ] ||        # index of the device
++  [ -z "${NETWORK_CARD_INDEX}" ] ||   # index of the network card
+   [ -z "${GW_IP_ADDRESS}" ] ||        # gateway ip address
+   [ -z "${DEVICE_IP_ADDRESS}" ] ||    # ip address to assign to the interface
+   [ -z "${CIDR_PREFIX_LENGTH}" ] ||   # the prefix length of the device IP cidr block
+@@ -40,7 +41,8 @@
+ fi
+
+ FILE="/etc/netplan/${DEVICE_NAME}.yaml"
+-ROUTE_TABLE="100${DEVICE_NUMBER}"
++SUFFIX=$NETWORK_CARD_INDEX$(printf "%02d" $DEVICE_NUMBER)
++ROUTE_TABLE="$(( $SUFFIX + 1000 ))"
+
+ echo "Configuring ${DEVICE_NAME} with IP:${DEVICE_IP_ADDRESS} CIDR_PREFIX:${CIDR_PREFIX_LENGTH} NETMASK:${NETMASK} GW:${GW_IP_ADDRESS} ROUTING_TABLE:${ROUTE_TABLE}"
+
+Index: cookbooks/aws-parallelcluster-environment/recipes/config/network_interfaces.rb
+IDEA additional info:
+Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
+<+>UTF-8
+===================================================================
+diff --git a/cookbooks/aws-parallelcluster-environment/recipes/config/network_interfaces.rb b/cookbooks/aws-parallelcluster-environment/recipes/config/network_interfaces.rb
+--- a/cookbooks/aws-parallelcluster-environment/recipes/config/network_interfaces.rb	(revision 9fe7e20db84ca556bcb830210c1f9ea852a38a89)
++++ b/cookbooks/aws-parallelcluster-environment/recipes/config/network_interfaces.rb	(revision cb83f4c721ead74d826b433e285eac80b88ad6dc)
+@@ -16,7 +16,13 @@
+ return if on_docker?
+
+ def network_card_index(mac, token)
++  # This IMDS call is not available on single NIC instance, therefore fallback to 0
+   uri = URI("http://169.254.169.254/latest/meta-data/network/interfaces/macs/#{mac}/network-card")
++  get_metadata_with_token(token, uri) || 0
++end
++
++def device_number(mac, token)
++  uri = URI("http://169.254.169.254/latest/meta-data/network/interfaces/macs/#{mac}/device-number")
+   get_metadata_with_token(token, uri)
+ end
+
+@@ -69,6 +75,7 @@
+   # Configure nw interfaces
+   macs.each do |mac|
+     device_name = device_name(mac)
++    device_number = device_number(mac, token)
+     network_card_index = network_card_index(mac, token)
+     gw_ip_address = gateway_address
+     device_ip_address = device_ip(mac, token)
+@@ -84,7 +91,8 @@
+       environment(
+         # TODO: The variables are a superset of what's required by individual scripts. Consider simplification.
+         'DEVICE_NAME' => device_name,
+-        'DEVICE_NUMBER' => "#{network_card_index}", # in configure_nw_interface DEVICE_NUMBER actually means network card index
++        'DEVICE_NUMBER' => "#{device_number}",
++        'NETWORK_CARD_INDEX' => "#{network_card_index}",
+         'GW_IP_ADDRESS' => gw_ip_address,
+         'DEVICE_IP_ADDRESS' => device_ip_address,
+         'CIDR_PREFIX_LENGTH' => cidr_prefix_length,
+EOF
+
+git apply support_secondary_network_interface_patch