fix(localdns): wait for resolv.conf update after networkctl reload to prevent race condition

parts/linux/cloud-init/artifacts/localdns.sh

@@ -388,6 +388,48 @@ add_iptable_rules_to_skip_conntrack_from_pods(){
     done
 }
 
+# Wait for DNS configuration to be applied after networkctl reload.
+# This function polls the resolv.conf to verify the expected DNS server is present.
+# Arguments:
+#   $1: expected_dns_ip - The DNS IP that should appear in resolv.conf.
+#   $2: should_contain - "true" if the IP should be present, "false" if it should be absent.
+#   $3: max_wait_seconds - Maximum time to wait for the change (default: 10).
+wait_for_dns_config_applied() {
+    local expected_dns_ip="$1"
+    local should_contain="$2"
+    local max_wait_seconds="${3:-10}"
+    local elapsed=0
+
+    echo "Waiting for DNS configuration to be applied (expecting ${expected_dns_ip} to be ${should_contain})..."
+
+    while [ "$elapsed" -lt "$max_wait_seconds" ]; do
+        # Get current DNS servers from resolv.conf.
+        local current_dns
+        current_dns=$(awk '/^nameserver/ {print $2}' "$RESOLV_CONF" 2>/dev/null | paste -sd' ')
+
+        if [ "$should_contain" = "true" ]; then
+            # Check if the expected DNS IP is present.
+            if echo "$current_dns" | grep -qw "$expected_dns_ip"; then
+                echo "DNS configuration applied successfully. Current DNS: ${current_dns}"
+                return 0
+            fi
+        else
+            # Check if the expected DNS IP is absent.
+            if ! echo "$current_dns" | grep -qw "$expected_dns_ip"; then
+                echo "DNS configuration reverted successfully. Current DNS: ${current_dns}"
+                return 0
+            fi
+        fi
+
+        sleep 1
+        elapsed=$((elapsed + 1))
+    done
+
+    echo "Timed out waiting for DNS configuration to be applied after ${max_wait_seconds} seconds."
+    echo "Expected ${expected_dns_ip} to be ${should_contain}, current DNS: $(awk '/^nameserver/ {print $2}' "$RESOLV_CONF" 2>/dev/null | paste -sd' ')"
+    return 1
+}
+
 # Disable DNS provided by DHCP and point the system at localdns.
 disable_dhcp_use_clusterlistener() {
     mkdir -p "${NETWORK_DROPIN_DIR}"
@@ -412,6 +454,14 @@ EOF
         echo "Failed to reload networkctl."
         return 1
     fi
+
+    # Wait for the DNS configuration to be applied.
+    # This ensures systemd-resolved has updated resolv.conf before we proceed.
+    if ! wait_for_dns_config_applied "${LOCALDNS_NODE_LISTENER_IP}" "true" 10; then
+        echo "Warning: DNS configuration may not have been fully applied."
+        return 1
+    fi
+
     return 0
 }
 
@@ -472,6 +522,14 @@ cleanup_iptables_and_dns() {
     fi
     echo "Reloading network configuration succeeded."
 
+    # Wait for the DNS configuration to be reverted.
+    # This ensures systemd-resolved has removed localdns from resolv.conf before we proceed.
+    # This is called both at startup (to clean up leftover state) and during shutdown.
+    if ! wait_for_dns_config_applied "${LOCALDNS_NODE_LISTENER_IP}" "false" 10; then
+        echo "Warning: DNS configuration may not have been fully reverted."
+        # Don't fail for this - the localdns IP might not have been configured previously.
+    fi
+
     return 0
 }
 

spec/parts/linux/cloud-init/artifacts/localdns_spec.sh

@@ -1110,4 +1110,112 @@ EOF
             The status should be success
         End
     End
+
+
+# This section tests - wait_for_dns_config_applied
+# This function is defined in parts/linux/cloud-init/artifacts/localdns.sh file.
+#------------------------------------------------------------------------------------------------------------------------------------
+    Describe 'wait_for_dns_config_applied'
+        setup() {
+            Include "./parts/linux/cloud-init/artifacts/localdns.sh"
+            TEST_DIR="/tmp/localdnstest"
+            RESOLV_CONF="${TEST_DIR}/run/systemd/resolve/resolv.conf"
+            mkdir -p "$(dirname "$RESOLV_CONF")"
+        }
+        cleanup() {
+            rm -rf "$TEST_DIR"
+        }
+        BeforeEach 'setup'
+        AfterEach 'cleanup'
+
+        #------------------------- wait_for_dns_config_applied (should_contain=true) ----------------------------------
+        It 'should return success immediately if expected IP is present (should_contain=true)'
+            cat > "$RESOLV_CONF" <<EOF
+nameserver 169.254.10.10
+EOF
+            When run wait_for_dns_config_applied "169.254.10.10" "true" 5
+            The status should be success
+            The stdout should include "DNS configuration applied successfully"
+            The stdout should include "Current DNS: 169.254.10.10"
+        End
+
+        It 'should return success if expected IP is present among multiple IPs (should_contain=true)'
+            cat > "$RESOLV_CONF" <<EOF
+nameserver 10.0.0.1
+nameserver 169.254.10.10
+nameserver 10.0.0.2
+EOF
+            When run wait_for_dns_config_applied "169.254.10.10" "true" 5
+            The status should be success
+            The stdout should include "DNS configuration applied successfully"
+        End
+
+        It 'should timeout if expected IP is not present (should_contain=true)'
+            cat > "$RESOLV_CONF" <<EOF
+nameserver 10.0.0.1
+nameserver 10.0.0.2
+EOF
+            When run wait_for_dns_config_applied "169.254.10.10" "true" 2
+            The status should be failure
+            The stdout should include "Timed out waiting for DNS configuration to be applied after 2 seconds"
+            The stdout should include "Expected 169.254.10.10 to be true"
+        End
+
+        #------------------------- wait_for_dns_config_applied (should_contain=false) ---------------------------------
+        It 'should return success immediately if expected IP is absent (should_contain=false)'
+            cat > "$RESOLV_CONF" <<EOF
+nameserver 10.0.0.1
+nameserver 10.0.0.2
+EOF
+            When run wait_for_dns_config_applied "169.254.10.10" "false" 5
+            The status should be success
+            The stdout should include "DNS configuration reverted successfully"
+        End
+
+        It 'should timeout if expected IP is still present (should_contain=false)'
+            cat > "$RESOLV_CONF" <<EOF
+nameserver 169.254.10.10
+nameserver 10.0.0.1
+EOF
+            When run wait_for_dns_config_applied "169.254.10.10" "false" 2
+            The status should be failure
+            The stdout should include "Timed out waiting for DNS configuration to be applied after 2 seconds"
+            The stdout should include "Expected 169.254.10.10 to be false"
+        End
+
+        It 'should return success if resolv.conf is empty (should_contain=false)'
+            > "$RESOLV_CONF"
+            When run wait_for_dns_config_applied "169.254.10.10" "false" 5
+            The status should be success
+            The stdout should include "DNS configuration reverted successfully"
+        End
+
+        #------------------------- wait_for_dns_config_applied (default timeout) --------------------------------------
+        It 'should use default timeout of 10 seconds when not specified'
+            cat > "$RESOLV_CONF" <<EOF
+nameserver 10.0.0.1
+EOF
+            # Note: This test checks behavior but uses a short actual wait by having the condition pass immediately
+            When run wait_for_dns_config_applied "10.0.0.1" "true"
+            The status should be success
+            The stdout should include "DNS configuration applied successfully"
+        End
+
+        #------------------------- wait_for_dns_config_applied (edge cases) -------------------------------------------
+        It 'should handle resolv.conf not existing gracefully'
+            rm -f "$RESOLV_CONF"
+            When run wait_for_dns_config_applied "169.254.10.10" "false" 2
+            The status should be success
+            The stdout should include "DNS configuration reverted successfully"
+        End
+
+        It 'should not match partial IP addresses'
+            cat > "$RESOLV_CONF" <<EOF
+nameserver 169.254.10.100
+EOF
+            When run wait_for_dns_config_applied "169.254.10.10" "true" 2
+            The status should be failure
+            The stdout should include "Timed out waiting for DNS configuration"
+        End
+    End
 End