feat: add udev rule to set default RX Buffer as 2048 on systems with 4 or more cores (#7665)

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com>
Co-authored-by: awesomenix <1703110+awesomenix@users.noreply.github.com>
Signed-off-by: Devin Wong <wongsiosun@outlook.com>

Author: 4 people

e2e/validation.go

@@ -43,6 +43,7 @@ func ValidateCommonLinux(ctx context.Context, s *Scenario) {
 	ValidateDiskQueueService(ctx, s)
 	ValidateLeakedSecrets(ctx, s)
 	ValidateIPTablesCompatibleWithCiliumEBPF(ctx, s)
+	ValidateRxBufferDefault(ctx, s)
 
 	ValidateSysctlConfig(ctx, s, map[string]string{
 		"net.ipv4.tcp_retries2":             "8",

e2e/validators.go

@@ -11,6 +11,7 @@ import (
 	"net"
 	"os"
 	"regexp"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -254,6 +255,90 @@ func ValidateSysctlConfig(ctx context.Context, s *Scenario, customSysctls map[st
 	}
 }
 
+// ValidateNetworkInterfaceConfig validates network interface configuration settings using ethtool.
+// It identifies network interfaces with slot names matching the enP* pattern (same logic as the udev rule),
+// then verifies that each interface has the expected configuration settings (e.g., rx buffer size).
+// The nicConfig map specifies the ethtool settings to validate (key: setting name, value: expected value).
+func ValidateNetworkInterfaceConfig(ctx context.Context, s *Scenario, nicConfig map[string]string) {
+	s.T.Helper()
+
+	// Get list of NICs using udevadm (same logic as udev rule)
+	getNicsCommand := []string{
+		"#!/usr/bin/env bash",
+		"set -euo pipefail",
+		"echo '=== NICs to Configure ==='",
+		"enp_ifaces=()",
+		"for dev in /sys/class/net/*; do",
+		"  iface=\"$(basename \"$dev\")\"",
+		"  slot=\"$(udevadm info -q property -p \"$dev\" 2>/dev/null | awk -F= '$1==\"ID_NET_NAME_SLOT\"{print $2; exit}')\"",
+		"  [[ \"$slot\" == enP* ]] && enp_ifaces+=(\"$iface\")",
+		"done",
+		"IFS=,; echo \"${enp_ifaces[*]}\"",
+	}
+	nicsResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(getNicsCommand, "\n"), 0, "could not get nics to configure")
+	s.T.Logf("NICs to configure:\n%s", nicsResult.stdout)
+
+	// Parse NIC output - it may be multi-line with header
+	lines := strings.Split(strings.TrimSpace(nicsResult.stdout), "\n")
+	nicsOutput := ""
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+		// Skip header lines
+		if strings.Contains(line, "===") || line == "" {
+			continue
+		}
+		nicsOutput = line
+		break
+	}
+
+	nics := strings.Split(nicsOutput, ",")
+
+	s.T.Logf("Parsed NICs list: %v (count: %d)", nics, len(nics))
+
+	if len(nics) == 0 || (len(nics) == 1 && strings.TrimSpace(nics[0]) == "") {
+		s.T.Fatalf("no nics found to validate network interface config")
+		return
+	}
+
+	for _, nic := range nics {
+		// Skip empty entries
+		nic = strings.TrimSpace(nic)
+		if nic == "" {
+			continue
+		}
+
+		s.T.Logf("Validating network interface config for NIC: %s", nic)
+
+		for setting, expectedValue := range nicConfig {
+			// Get full ethtool output for debugging
+			debugCommand := []string{
+				"set -ex",
+				fmt.Sprintf("echo '=== Full ethtool output for %s ==='", nic),
+				fmt.Sprintf("sudo ethtool -g %s", nic),
+			}
+			debugResult := execScriptOnVMForScenario(ctx, s, strings.Join(debugCommand, "\n"))
+			s.T.Logf("Full ethtool output for %s:\n%s", nic, debugResult.stdout)
+
+			command := []string{
+				"set -ex",
+				fmt.Sprintf("sudo ethtool -g %s | grep -A 5 'Current hardware settings' | grep -i %s: | awk '{print $2}'", nic, setting),
+			}
+			execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "could not get ethtool config")
+			actualValue := strings.TrimSpace(execResult.stdout)
+			s.T.Logf("Ethtool setting %s for NIC %s: expected=%s, actual=%s", setting, nic, expectedValue, actualValue)
+			require.Equal(s.T, expectedValue, actualValue, "expected %s to be %s on nic %s, but got %s.\nFull ethtool output:\n%s", setting, expectedValue, nic, actualValue, debugResult.stdout)
+		}
+	}
+}
+
+// ValidateAzureNetworkFiles checks that udev rules files exist.
+func ValidateAzureNetworkFiles(ctx context.Context, s *Scenario) {
+	s.T.Helper()
+
+	ValidateFileExists(ctx, s, "/opt/azure-network/configure-azure-network.sh")
+	ValidateFileExists(ctx, s, "/etc/udev/rules.d/99-azure-network.rules")
+}
+
 func ValidateNvidiaSMINotInstalled(ctx context.Context, s *Scenario) {
 	s.T.Helper()
 	command := []string{
@@ -1623,3 +1708,35 @@ func ValidateNodeHasLabel(ctx context.Context, s *Scenario, labelKey, expectedVa
 	require.True(s.T, exists, "expected node %q to have label %q, but it was not found", s.Runtime.VM.KubeName, labelKey)
 	require.Equal(s.T, expectedValue, actualValue, "expected node %q label %q to have value %q, but got %q", s.Runtime.VM.KubeName, labelKey, expectedValue, actualValue)
 }
+
+// ValidateRxBufferDefault validates rx buffer config using default values based on VM's CPU count
+func ValidateRxBufferDefault(ctx context.Context, s *Scenario) {
+	s.T.Helper()
+
+	// Query the VM's actual CPU count using nproc
+	cpuCountCmd := "nproc"
+	result := execScriptOnVMForScenarioValidateExitCode(ctx, s, cpuCountCmd, 0, "could not get CPU count from VM")
+	vmCPUCount := strings.TrimSpace(result.stdout)
+
+	// Parse CPU count
+	cpuCount, err := strconv.Atoi(vmCPUCount)
+	require.NoError(s.T, err, "failed to parse CPU count: %s", vmCPUCount)
+
+	// Determine expected rx based on VM's CPU count (matching configure-azure-network.sh logic)
+	expectedRx := "1024"
+	if cpuCount >= 4 {
+		expectedRx = "2048"
+	}
+
+	s.T.Logf("VM has %d CPUs, expecting rx buffer size: %s", cpuCount, expectedRx)
+
+	customNicConfig := map[string]string{
+		"rx": expectedRx,
+	}
+
+	// Validate files exist
+	ValidateAzureNetworkFiles(ctx, s)
+
+	// Validate network interface settings match expected default
+	ValidateNetworkInterfaceConfig(ctx, s, customNicConfig)
+}

parts/linux/cloud-init/artifacts/99-azure-network.rules

@@ -0,0 +1,5 @@
+# AKS Network Tuning udev rule
+# This rule configures network interface RX ring buffer settings
+# Applies to network interfaces with slot names (ID_NET_NAME_SLOT) matching enP* pattern
+SUBSYSTEM=="net", ACTION=="add", ENV{ID_NET_NAME_SLOT}=="enP*", RUN+="/opt/azure-network/configure-azure-network.sh %k"
+SUBSYSTEM=="net", ACTION=="change", ENV{ID_NET_NAME_SLOT}=="enP*", RUN+="/opt/azure-network/configure-azure-network.sh %k"

parts/linux/cloud-init/artifacts/configure-azure-network.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# This script configures network interface settings for Azure NICs.
+# Called by udev with interface name as argument
+
+INTERFACE="$1"
+
+# Exit if no interface provided
+if [ -z "$INTERFACE" ]; then
+    echo "No interface provided, exiting"
+    exit 0
+fi
+
+# Check if interface exists
+if [ ! -d "/sys/class/net/$INTERFACE" ]; then
+    echo "NIC $INTERFACE does not exist. Skipping."
+    exit 0
+fi
+
+# Determine default RX buffer size based on number of CPUs
+NUM_CPUS=$(nproc)
+if [ "$NUM_CPUS" -ge 4 ]; then
+    DEFAULT_RX_BUFFER_SIZE=2048
+else
+    DEFAULT_RX_BUFFER_SIZE=1024
+fi
+
+# Get current RX buffer size
+CURRENT_RX=$(ethtool -g "$INTERFACE" 2>/dev/null | grep -A4 "Current hardware settings" | grep "^RX:" | awk '{print $2}')
+
+# Only proceed if current RX is 1024
+if [ "$CURRENT_RX" != "1024" ]; then
+    echo "Current RX buffer size is $CURRENT_RX (not 1024), skipping configuration for $INTERFACE"
+    exit 0
+fi
+
+# Use default unless overridden by config file
+RX_SIZE=$DEFAULT_RX_BUFFER_SIZE
+
+echo "Detected $NUM_CPUS CPUs, current RX is 1024, configuring $INTERFACE with rx=$RX_SIZE"
+ethtool -G "$INTERFACE" rx "$RX_SIZE" || echo "Failed to set ring parameters for $INTERFACE"

parts/linux/cloud-init/artifacts/cse_config.sh

@@ -817,6 +817,18 @@ ensureSysctl() {
     retrycmd_if_failure 24 5 25 sysctl --system
 }
 
+ensureAzureNetworkConfig() {
+    # Reload udev rules to pick up the new azure-network rules
+    udevadm control --reload-rules
+
+    # Trigger udev to detect and populate network interfaces
+    echo "Triggering udev for network devices..."
+    udevadm trigger --subsystem-match=net --action=add
+
+    # Give udev time to process and trigger the systemd service
+    udevadm settle --timeout=10
+}
+
 ensureK8sControlPlane() {
     if $REBOOTREQUIRED || [ "$NO_OUTBOUND" = "true" ]; then
         return

parts/linux/cloud-init/artifacts/cse_main.sh

@@ -340,6 +340,9 @@ function nodePrep {
         touch /opt/azure/outbound-check-skipped
     fi
 
+    # Configure Azure network settings (udev rules for NIC configuration)
+    logs_to_events "AKS.CSE.ensureAzureNetworkConfig" ensureAzureNetworkConfig
+
     # Determine if GPU driver installation should be skipped
     export -f should_skip_nvidia_drivers
     skip_nvidia_driver_install=$(should_skip_nvidia_drivers)

parts/linux/cloud-init/nodecustomdata.yml

@@ -321,6 +321,20 @@ write_files:
   content: !!binary |
     {{GetVariableProperty "cloudInitData" "validateKubeletCredentialsScript"}}
 
+- path: /opt/azure-network/configure-azure-network.sh
+  permissions: "0755"
+  encoding: gzip
+  owner: root
+  content: !!binary |
+    {{GetVariableProperty "cloudInitData" "configureAzureNetworkScript"}}
+
+- path: /etc/udev/rules.d/99-azure-network.rules
+  permissions: "0644"
+  encoding: gzip
+  owner: root
+  content: !!binary |
+    {{GetVariableProperty "cloudInitData" "azureNetworkUdevRule"}}
+
 - path: /etc/kubernetes/certs/ca.crt
   permissions: "0600"
   encoding: base64

pkg/agent/const.go

@@ -96,6 +96,10 @@ const (
 	ensureNoDupEbtablesScript  = "linux/cloud-init/artifacts/ensure-no-dup.sh"
 	ensureNoDupEbtablesService = "linux/cloud-init/artifacts/ensure-no-dup.service"
 
+	// Azure network configuration files.
+	configureAzureNetworkScript = "linux/cloud-init/artifacts/configure-azure-network.sh"
+	azureNetworkUdevRule        = "linux/cloud-init/artifacts/99-azure-network.rules"
+
 	componentManifestFile = "linux/cloud-init/artifacts/manifest.json"
 )