From 34cab0e2a051faa1d950affb1ceb3811a19c2db4 Mon Sep 17 00:00:00 2001 From: Mat Kowalski Date: Wed, 15 Oct 2025 16:45:50 +0200 Subject: [PATCH] OCPBUGS-63152: Fail wait-for-node-ip if primary IP(s) disappear Under specific scenario it is possible that between nodeip-configuration.service and kubelet.service the br-ex interface loses one of its IP stacks. This is undesired because it will cause kubelet to be stuck without any clear error message. With this change we will make wait-for-node-ip.service fail in a clear way if one of the IP addresses used by kubelet are not available. The timeout 60 seconds has been selected as arbitrary. In a positive scenario the service should succeed immediately. In a negative scenario when the IP address really disappeared, 60 seconds is fair to wait. In a scenario when we are racing and handle tentative IPv6 addresses 60 seconds is enough for an address to settle and become permanent. --- .../common/_base/files/wait-for-node-ip.yaml | 56 +++++++++++++++++++ .../_base/files/wait-for-primary-ip.yaml | 31 ---------- .../_base/units/mtu-migration.service.yaml | 2 +- .../_base/units/wait-for-primary-ip.yaml | 6 +- 4 files changed, 60 insertions(+), 35 deletions(-) create mode 100644 templates/common/_base/files/wait-for-node-ip.yaml delete mode 100644 templates/common/_base/files/wait-for-primary-ip.yaml diff --git a/templates/common/_base/files/wait-for-node-ip.yaml b/templates/common/_base/files/wait-for-node-ip.yaml new file mode 100644 index 0000000000..17e92aabd6 --- /dev/null +++ b/templates/common/_base/files/wait-for-node-ip.yaml @@ -0,0 +1,56 @@ +mode: 0755 +path: "/usr/local/bin/wait-for-node-ip.sh" +contents: + inline: | + #!/bin/bash + set -eux + + wait_for_ip_bind() { + local ip=$1 + local end_time=$((SECONDS + 60)) + while [[ $SECONDS -lt $end_time ]] + do + random_port=$(shuf -i 50000-60000 -n 1) + echo "Trying to bind ${ip} on port ${random_port}" + exit_code=$(timeout 2s nc -l "${ip}" ${random_port}; echo $?) + if [[ exit_code -eq 124 ]]; then + echo "Address bound successfully" + exit 0 + fi + sleep 10 + done + echo "Failed to bind ${ip} after 60 seconds" + return 123 + } + + if [ ! -e /etc/nmstate/openshift/applied ]; then + # No need to do this if no NMState configuration was applied + exit 0 + fi + + # This logic is borrowed from configure-ovs.sh + # TODO: Find a platform-agnostic way to do this. It won't work on platforms where + # nodeip-configuration is not used. + ip=$(cat /run/nodeip-configuration/primary-ip) + if [[ "${ip}" == "" ]]; then + echo "No primary ip to bind was found" + exit 1 + fi + wait_for_ip_bind "${ip}" + + # Do not fail below because we don't know if we run on primary-v4 or primary-v6 platform. + # We only want to make sure that if nodeip-configuration detected an IP address, the + # address is usable. + ip=$(cat /run/nodeip-configuration/ipv4) + if [[ "${ip}" == "" ]]; then + echo "No ipv4 to bind was found" + else + wait_for_ip_bind "${ip}" + fi + + ip=$(cat /run/nodeip-configuration/ipv6) + if [[ "${ip}" == "" ]]; then + echo "No ipv6 to bind was found" + else + wait_for_ip_bind "${ip}" + fi diff --git a/templates/common/_base/files/wait-for-primary-ip.yaml b/templates/common/_base/files/wait-for-primary-ip.yaml deleted file mode 100644 index b80b1776cb..0000000000 --- a/templates/common/_base/files/wait-for-primary-ip.yaml +++ /dev/null @@ -1,31 +0,0 @@ -mode: 0755 -path: "/usr/local/bin/wait-for-primary-ip.sh" -contents: - inline: | - #!/bin/bash - set -eux - - if [ ! -e /etc/nmstate/openshift/applied ]; then - # No need to do this if no NMState configuration was applied - exit 0 - fi - - # This logic is borrowed from configure-ovs.sh - # TODO: Find a platform-agnostic way to do this. It won't work on platforms where - # nodeip-configuration is not used. - ip=$(cat /run/nodeip-configuration/primary-ip) - if [[ "${ip}" == "" ]]; then - echo "No ip to bind was found" - exit 1 - fi - while : - do - random_port=$(shuf -i 50000-60000 -n 1) - echo "Trying to bind ${ip} on port ${random_port}" - exit_code=$(timeout 2s nc -l "${ip}" ${random_port}; echo $?) - if [[ exit_code -eq 124 ]]; then - echo "Address bound successfully" - exit 0 - fi - sleep 10 - done diff --git a/templates/common/_base/units/mtu-migration.service.yaml b/templates/common/_base/units/mtu-migration.service.yaml index e7c12ff66c..72d4cbd806 100644 --- a/templates/common/_base/units/mtu-migration.service.yaml +++ b/templates/common/_base/units/mtu-migration.service.yaml @@ -6,7 +6,7 @@ contents: | Description=Configures interfaces and routes with temporary MTUs during MTU migration Requires=openvswitch.service ovs-configuration.service Wants=NetworkManager-wait-online.service - After=NetworkManager-wait-online.service openvswitch.service network.service ovs-configuration.service wait-for-primary-ip.service + After=NetworkManager-wait-online.service openvswitch.service network.service ovs-configuration.service wait-for-node-ip.service Before=kubelet-dependencies.target node-valid-hostname.service [Service] diff --git a/templates/common/_base/units/wait-for-primary-ip.yaml b/templates/common/_base/units/wait-for-primary-ip.yaml index 461b226be6..bba4cabff2 100644 --- a/templates/common/_base/units/wait-for-primary-ip.yaml +++ b/templates/common/_base/units/wait-for-primary-ip.yaml @@ -1,8 +1,8 @@ -name: wait-for-primary-ip.service +name: wait-for-node-ip.service enabled: true contents: | [Unit] - Description=Ensure primary IP is assigned and usable + Description=Ensure node IP(s) are assigned and usable Requires=nmstate.service After=nmstate.service Before=kubelet-dependencies.target @@ -14,7 +14,7 @@ contents: | # available in systemd v244 and higher. ExecStart=/bin/bash -c " \ until \ - /usr/local/bin/wait-for-primary-ip.sh; \ + /usr/local/bin/wait-for-node-ip.sh; \ do \ sleep 10; \ done"