feat: enable localdns hosts plugin to cache critical AKS FQDNs

aks-node-controller/proto/aksnodeconfig/v1/localdns_config.proto

@@ -19,6 +19,9 @@ message LocalDnsProfile {
 
   // KubeDns overrides apply to DNS traffic from pods with dnsPolicy:ClusterFirst (referred to as KubeDns traffic).
   map<string, LocalDnsOverrides> kube_dns_overrides = 5;
+
+  // Field 6 was critical_hosts_entries, removed.
+  reserved 6;
 }
 
 // Represents DNS override settings for both VnetDNS and KubeDNS traffic.

e2e/validation.go

@@ -69,10 +69,20 @@ func ValidateCommonLinux(ctx context.Context, s *Scenario) {
 		ValidateKubeletNodeIP(ctx, s)
 	}
 
-	// localdns is not supported on scriptless, privatekube and VHDUbuntu2204Gen2ContainerdAirgappedK8sNotCached.
+	// localdns is not supported on FIPS VHDs, older VHDs (privatekube, airgapped), and AzureLinux OSGuard.
 	if !s.VHD.UnsupportedLocalDns {
 		ValidateLocalDNSService(ctx, s, "enabled")
 		ValidateLocalDNSResolution(ctx, s, "169.254.10.10")
+		// Validate aks-hosts-setup service ran successfully and timer is active
+		ValidateAKSHostsSetupService(ctx, s)
+		// Validate hosts file contains resolved IPs for critical FQDNs (IPs resolved dynamically)
+		ValidateLocalDNSHostsFile(ctx, s, []string{
+			"mcr.microsoft.com",
+			"login.microsoftonline.com",
+			"acs-mirror.azureedge.net",
+		})
+		// Validate localdns resolves fake FQDN from hosts file (proves hosts plugin bypass)
+		ValidateLocalDNSHostsPluginBypass(ctx, s)
 	}
 
 	ValidateInspektorGadget(ctx, s)

e2e/validators.go

@@ -1384,6 +1384,172 @@ func ValidateLocalDNSResolution(ctx context.Context, s *Scenario, server string)
 	assert.Contains(s.T, execResult.stdout, fmt.Sprintf("SERVER: %s", server))
 }
 
+// ValidateLocalDNSHostsFile checks that /etc/localdns/hosts contains entries for critical FQDNs.
+// It dynamically resolves IPs on the VM and verifies they match what's in the hosts file.
+// This avoids hardcoding IPs that can change over time.
+func ValidateLocalDNSHostsFile(ctx context.Context, s *Scenario, fqdns []string) {
+	s.T.Helper()
+
+	// Build script that resolves each FQDN and checks it exists in hosts file
+	script := fmt.Sprintf(`set -euo pipefail
+hosts_file="/etc/localdns/hosts"
+fqdns=(%s)
+
+echo "=== Validating /etc/localdns/hosts contains resolved IPs for critical FQDNs ==="
+echo ""
+echo "Current hosts file contents:"
+cat "$hosts_file"
+echo ""
+
+errors=0
+for fqdn in "${fqdns[@]}"; do
+    echo "Checking FQDN: $fqdn"
+
+    # Resolve IPv4 addresses using the Azure DNS (168.63.129.16)
+    ipv4_addrs=$(nslookup -type=A "$fqdn" 168.63.129.16 2>/dev/null | awk '/^Address: / && !/^Address: .*#/ {print $2}' | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$' || true)
+
+    if [ -z "$ipv4_addrs" ]; then
+        echo "  WARNING: Could not resolve IPv4 for $fqdn, skipping IP validation"
+        # At minimum, check the FQDN exists in the file
+        if ! grep -qF "$fqdn" "$hosts_file"; then
+            echo "  ERROR: FQDN $fqdn not found in hosts file at all"
+            errors=$((errors + 1))
+        fi
+        continue
+    fi
+
+    # Check each resolved IP exists in the hosts file for this FQDN
+    for ip in $ipv4_addrs; do
+        expected_entry="$ip $fqdn"
+        if grep -qF "$expected_entry" "$hosts_file"; then
+            echo "  OK: Found '$expected_entry' in hosts file"
+        else
+            echo "  ERROR: Expected entry '$expected_entry' not found in hosts file"
+            errors=$((errors + 1))
+        fi
+    done
+done
+
+echo ""
+if [ $errors -gt 0 ]; then
+    echo "FAILED: $errors entries missing from hosts file"
+    exit 1
+else
+    echo "SUCCESS: All critical FQDNs have correct entries in hosts file"
+    exit 0
+fi
+`, quoteFQDNsForBash(fqdns))
+
+	execScriptOnVMForScenarioValidateExitCode(ctx, s, script, 0,
+		"hosts file should contain resolved IPs for critical FQDNs")
+}
+
+// quoteFQDNsForBash converts a slice of FQDNs to a bash array string
+func quoteFQDNsForBash(fqdns []string) string {
+	quoted := make([]string, len(fqdns))
+	for i, fqdn := range fqdns {
+		quoted[i] = fmt.Sprintf("%q", fqdn)
+	}
+	return strings.Join(quoted, " ")
+}
+
+// ValidateAKSHostsSetupService checks that aks-hosts-setup.service ran successfully
+// and the aks-hosts-setup.timer is active to ensure periodic refresh of /etc/localdns/hosts.
+func ValidateAKSHostsSetupService(ctx context.Context, s *Scenario) {
+	s.T.Helper()
+
+	// Check that aks-hosts-setup.service completed successfully (oneshot service)
+	serviceScript := `set -euo pipefail
+svc="aks-hosts-setup.service"
+# For oneshot services, check if it ran successfully (exit code 0)
+result=$(systemctl show -p Result "$svc" --value 2>/dev/null || echo "unknown")
+echo "aks-hosts-setup.service result: $result"
+if [ "$result" != "success" ]; then
+    echo "ERROR: aks-hosts-setup.service did not complete successfully"
+    systemctl status "$svc" --no-pager || true
+    journalctl -u "$svc" --no-pager -n 50 || true
+    exit 1
+fi
+`
+	execScriptOnVMForScenarioValidateExitCode(ctx, s, serviceScript, 0,
+		"aks-hosts-setup.service should have completed successfully")
+
+	// Check that aks-hosts-setup.timer is active for periodic refresh
+	timerScript := `set -euo pipefail
+timer="aks-hosts-setup.timer"
+active=$(systemctl is-active "$timer" 2>/dev/null || true)
+echo "aks-hosts-setup.timer: active=$active"
+if [ "$active" != "active" ]; then
+    echo "ERROR: aks-hosts-setup.timer is not active"
+    systemctl status "$timer" --no-pager || true
+    exit 1
+fi
+`
+	execScriptOnVMForScenarioValidateExitCode(ctx, s, timerScript, 0,
+		"aks-hosts-setup.timer should be active for periodic hosts file refresh")
+}
+
+// ValidateLocalDNSHostsPluginBypass verifies that localdns resolves FQDNs from /etc/localdns/hosts
+// without querying the upstream DNS server. This confirms the hosts plugin is working correctly.
+// It injects a fake FQDN (that doesn't exist in public DNS) into the hosts file and verifies
+// localdns can resolve it - proving the hosts plugin is functioning.
+func ValidateLocalDNSHostsPluginBypass(ctx context.Context, s *Scenario) {
+	s.T.Helper()
+
+	// Use a fake FQDN that doesn't exist in public DNS and a TEST-NET-3 IP (RFC 5737)
+	// If this resolves, it MUST be coming from the hosts file
+	fakeFQDN := "fake-mcr.microsoft.com"
+	fakeIP := "203.0.113.42"
+
+	script := fmt.Sprintf(`set -euo pipefail
+fake_fqdn=%q
+fake_ip=%q
+hosts_file="/etc/localdns/hosts"
+
+echo "=== Testing localdns hosts plugin bypass ==="
+echo "Injecting fake entry: $fake_ip $fake_fqdn"
+
+# Add fake entry to hosts file
+echo "$fake_ip $fake_fqdn" | sudo tee -a "$hosts_file" > /dev/null
+
+echo "Current hosts file contents:"
+sudo cat "$hosts_file"
+
+# Give localdns a moment to pick up the change (hosts plugin reloads periodically)
+sleep 2
+
+echo ""
+echo "Querying localdns for fake FQDN: $fake_fqdn"
+echo "If this resolves to $fake_ip, it proves the hosts plugin is working"
+
+# Query localdns at the cluster listener IP
+result=$(dig "$fake_fqdn" @169.254.10.10 +short +timeout=5 +tries=2 2>/dev/null || true)
+echo "DNS response: $result"
+
+# Check if the fake IP is in the response
+if echo "$result" | grep -q "$fake_ip"; then
+    echo ""
+    echo "SUCCESS: localdns resolved fake FQDN $fake_fqdn to $fake_ip from hosts file!"
+    echo "This proves the hosts plugin is correctly bypassing upstream DNS."
+    exit 0
+else
+    echo ""
+    echo "ERROR: Expected fake IP $fake_ip not found in response for $fake_fqdn"
+    echo "The hosts plugin may not be working correctly."
+    echo ""
+    echo "Full dig output:"
+    dig "$fake_fqdn" @169.254.10.10 +timeout=5 +tries=2 || true
+    echo ""
+    echo "localdns service status:"
+    systemctl status localdns --no-pager -n 10 || true
+    exit 1
+fi
+`, fakeFQDN, fakeIP)
+
+	execScriptOnVMForScenarioValidateExitCode(ctx, s, script, 0,
+		"localdns should resolve fake FQDN from hosts file (proving hosts plugin bypass)")
+}
+
 // ValidateJournalctlOutput checks if specific content exists in the systemd service logs
 func ValidateJournalctlOutput(ctx context.Context, s *Scenario, serviceName string, expectedContent string) {
 	s.T.Helper()

parts/linux/cloud-init/artifacts/aks-hosts-setup.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Populate /etc/localdns/hosts with critical AKS FQDN addresses
+After=network-online.target
+Wants=network-online.target
+Before=kubelet.service localdns.service
+
+[Service]
+Type=oneshot
+ExecStart=/opt/azure/containers/aks-hosts-setup.sh
+
+[Install]
+WantedBy=multi-user.target kubelet.service

parts/linux/cloud-init/artifacts/aks-hosts-setup.sh

@@ -0,0 +1,98 @@
+#!/bin/bash
+set -uo pipefail
+
+# aks-hosts-setup.sh
+# Resolves A and AAAA records for critical AKS FQDNs and populates /etc/localdns/hosts
+
+HOSTS_FILE="/etc/localdns/hosts"
+
+# Ensure the directory exists
+mkdir -p "$(dirname "$HOSTS_FILE")"
+
+# Critical AKS FQDNs that should be cached for DNS reliability
+CRITICAL_FQDNS=(
+    "acs-mirror.azureedge.net"
+    "eastus.data.mcr.microsoft.com"
+    "login.microsoftonline.com"
+    "management.azure.com"
+    "mcr.microsoft.com"
+    "packages.aks.azure.com"
+    "packages.microsoft.com"
+)
+
+# Function to resolve IPv4 addresses for a domain
+# Filters output to only include valid IPv4 addresses (rejects NXDOMAIN, SERVFAIL, hostnames, etc.)
+resolve_ipv4() {
+    local domain="$1"
+    local output
+    output=$(nslookup -type=A "${domain}" 2>/dev/null) || return 0
+    # Parse Address lines (skip server address with #), validate IPv4 format (4 octets of 1-3 digits)
+    echo "${output}" | awk '/^Address: / && !/^Address: .*#/ {print $2}' | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$' || return 0
+}
+
+# Function to resolve IPv6 addresses for a domain
+# Filters output to only include valid IPv6 addresses (rejects NXDOMAIN, SERVFAIL, hostnames, etc.)
+resolve_ipv6() {
+    local domain="$1"
+    local output
+    output=$(nslookup -type=AAAA "${domain}" 2>/dev/null) || return 0
+    # Parse Address lines (skip server address with #), validate IPv6 format (must contain : and only hex/colons, min 3 chars)
+    echo "${output}" | awk '/^Address: / && !/^Address: .*#/ {print $2}' | grep -E '^[0-9a-fA-F:]{3,}$' | grep ':' || return 0
+}
+
+echo "Starting AKS critical FQDN hosts resolution at $(date)"
+
+# Track if we resolved at least one address
+RESOLVED_ANY=false
+
+# Start building the hosts file content
+HOSTS_CONTENT="# AKS critical FQDN addresses resolved at $(date)
+# This file is automatically generated by aks-hosts-setup.service
+"
+
+# Resolve each FQDN
+for DOMAIN in "${CRITICAL_FQDNS[@]}"; do
+    echo "Resolving addresses for ${DOMAIN}..."
+
+    # Get IPv4 and IPv6 addresses using helper functions
+    IPV4_ADDRS=$(resolve_ipv4 "${DOMAIN}")
+    IPV6_ADDRS=$(resolve_ipv6 "${DOMAIN}")
+
+    # Check if we got any results for this domain
+    if [ -z "${IPV4_ADDRS}" ] && [ -z "${IPV6_ADDRS}" ]; then
+        echo "  WARNING: No IP addresses resolved for ${DOMAIN}"
+        continue
+    fi
+
+    RESOLVED_ANY=true
+    HOSTS_CONTENT+="
+# ${DOMAIN}"
+
+    if [ -n "${IPV4_ADDRS}" ]; then
+        for addr in ${IPV4_ADDRS}; do
+            HOSTS_CONTENT+="
+${addr} ${DOMAIN}"
+        done
+    fi
+
+    if [ -n "${IPV6_ADDRS}" ]; then
+        for addr in ${IPV6_ADDRS}; do
+            HOSTS_CONTENT+="
+${addr} ${DOMAIN}"
+        done
+    fi
+done
+
+# Check if we resolved at least one domain
+if [ "${RESOLVED_ANY}" != "true" ]; then
+    echo "WARNING: No IP addresses resolved for any domain at $(date)"
+    echo "This is likely a temporary DNS issue. Timer will retry later."
+    # Keep existing hosts file intact and exit successfully so systemd doesn't mark unit as failed
+    exit 0
+fi
+
+# Write the hosts file
+echo "Writing addresses to ${HOSTS_FILE}..."
+echo "${HOSTS_CONTENT}" > "${HOSTS_FILE}"
+
+echo "AKS critical FQDN hosts resolution completed at $(date)"

parts/linux/cloud-init/artifacts/aks-hosts-setup.timer

@@ -0,0 +1,16 @@
+[Unit]
+Description=Run AKS hosts setup periodically
+Before=localdns.service
+
+[Timer]
+# Run immediately on boot
+OnBootSec=0
+# Run 15 minutes after the last activation (AKS critical FQDN IPs don't change frequently)
+OnUnitActiveSec=15min
+# Timer accuracy (how much systemd can delay)
+AccuracySec=1s
+# Add randomization to avoid thundering herd if multiple nodes boot simultaneously
+RandomizedDelaySec=60s
+
+[Install]
+WantedBy=timers.target

parts/linux/cloud-init/artifacts/cse_config.sh

@@ -1194,6 +1194,23 @@ enableLocalDNS() {
     echo "Enable localdns succeeded."
 }
 
+# This function enables and starts the aks-hosts-setup timer.
+# The timer periodically resolves critical AKS FQDN DNS records and populates /etc/localdns/hosts.
+enableAKSHostsSetup() {
+    local hosts_file="/etc/localdns/hosts"
+
+    # Run the script once immediately to resolve live DNS before kubelet starts
+    echo "Running initial aks-hosts-setup to resolve DNS..."
+    mkdir -p "$(dirname "${hosts_file}")"
+    /opt/azure/containers/aks-hosts-setup.sh || echo "Warning: Initial hosts setup failed"
+
+    # Enable the timer for periodic refresh (every 15 minutes)
+    # This will update the hosts file with fresh IPs from live DNS
+    echo "Enabling aks-hosts-setup timer..."
+    systemctlEnableAndStart aks-hosts-setup.timer 30 || exit $ERR_SYSTEMCTL_START_FAIL
+    echo "aks-hosts-setup timer enabled successfully."
+}
+
 configureManagedGPUExperience() {
     if [ "${GPU_NODE}" != "true" ] || [ "${skip_nvidia_driver_install}" = "true" ]; then
         return

parts/linux/cloud-init/artifacts/cse_main.sh

@@ -297,6 +297,11 @@ EOF
 
     # This is to enable localdns using scriptless.
     if [ "${SHOULD_ENABLE_LOCALDNS}" = "true" ]; then
+        # Write hosts file BEFORE starting LocalDNS so it has entries to serve
+        # Enable aks-hosts-setup timer to periodically resolve and cache critical AKS FQDN DNS addresses
+        logs_to_events "AKS.CSE.enableAKSHostsSetup" enableAKSHostsSetup || exit $ERR_SYSTEMCTL_START_FAIL
+
+        # Start LocalDNS after hosts file is populated
         logs_to_events "AKS.CSE.enableLocalDNS" enableLocalDNS || exit $ERR_LOCALDNS_FAIL
     fi
 

pkg/agent/baker.go

@@ -1888,6 +1888,12 @@ health-check.localdns.local:53 {
     {{- end }}
     bind {{$.NodeListenerIP}}
     {{- if $isRootDomain}}
+    # Check /etc/localdns/hosts first for critical AKS FQDNs (mcr.microsoft.com, packages.aks.azure.com, etc.)
+    hosts /etc/localdns/hosts {
+        fallthrough
+    }
+    {{- end}}
+    {{- if $isRootDomain}}
     forward . {{$.AzureDNSIP}} {
     {{- else}}
     {{- if $fwdToClusterCoreDNS}}
@@ -1948,6 +1954,12 @@ health-check.localdns.local:53 {
     log
     {{- end }}
     bind {{$.ClusterListenerIP}}
+    {{- if $isRootDomain}}
+    # Check /etc/localdns/hosts first for critical AKS FQDNs (mcr.microsoft.com, packages.aks.azure.com, etc.)
+    hosts /etc/localdns/hosts {
+        fallthrough
+    }
+    {{- end}}
     {{- if $fwdToClusterCoreDNS}}
     forward . {{$.CoreDNSServiceIP}} {
     {{- else}}