For NSX setup, it is possible that VirtualNetwork is not defined for a given VM service VM. (#3446)

skogta · web-flow · commit fff2b835f298 · 2025-07-31T17:43:14.000-07:00
In that case, CSI should rely on the NamespaceNetworkInfo CR's SNAT IP to determine the VM's external IP.
diff --git a/manifests/supervisorcluster/1.29/cns-csi.yaml b/manifests/supervisorcluster/1.29/cns-csi.yaml
@@ -117,6 +117,9 @@ rules:
   - apiGroups: ["encryption.vmware.com"]
     resources: ["encryptionclasses"]
     verbs: ["get", "list", "watch"]
+  - apiGroups: ["nsx.vmware.com"]
+    resources: ["namespacenetworkinfos"]
+    verbs: ["get", "list"]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
diff --git a/manifests/supervisorcluster/1.30/cns-csi.yaml b/manifests/supervisorcluster/1.30/cns-csi.yaml
@@ -120,6 +120,9 @@ rules:
   - apiGroups: ["cluster.x-k8s.io"]
     resources: ["clusters"]
     verbs: ["get", "list", "watch"]
+  - apiGroups: ["nsx.vmware.com"]
+    resources: ["namespacenetworkinfos"]
+    verbs: ["get", "list"]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
diff --git a/manifests/supervisorcluster/1.31/cns-csi.yaml b/manifests/supervisorcluster/1.31/cns-csi.yaml
@@ -120,6 +120,9 @@ rules:
   - apiGroups: ["cluster.x-k8s.io"]
     resources: ["clusters"]
     verbs: ["get", "list", "watch"]
+  - apiGroups: ["nsx.vmware.com"]
+    resources: ["namespacenetworkinfos"]
+    verbs: ["get", "list"]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
diff --git a/manifests/supervisorcluster/1.32/cns-csi.yaml b/manifests/supervisorcluster/1.32/cns-csi.yaml
@@ -120,6 +120,9 @@ rules:
   - apiGroups: ["cluster.x-k8s.io"]
     resources: ["clusters"]
     verbs: ["get", "list", "watch"]
+  - apiGroups: ["nsx.vmware.com"]
+    resources: ["namespacenetworkinfos"]
+    verbs: ["get", "list"]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
diff --git a/pkg/syncer/cnsoperator/util/util.go b/pkg/syncer/cnsoperator/util/util.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2021 The Kubernetes Authors.
+Copyright 2021-2025 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -37,6 +37,8 @@ import (
 
 	vimtypes "github.com/vmware/govmomi/vim25/types"
 	"sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/config"
+	"sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service/common"
+	"sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service/common/commonco"
 	"sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service/logger"
 )
 
@@ -52,6 +54,12 @@ var networkInfoGVR = schema.GroupVersionResource{
 	Resource: "networkinfos",
 }
 
+var namespaceNetworkInfoGVR = schema.GroupVersionResource{
+	Group:    "nsx.vmware.com",
+	Version:  "v1alpha1",
+	Resource: "namespacenetworkinfos",
+}
+
 const (
 	snatIPAnnotation = "ncp/snat_ip"
 	// Namespace for system resources.
@@ -113,9 +121,16 @@ func GetTKGVMIP(ctx context.Context, vmOperatorClient client.Client, dc dynamic.
 		return "", err
 	}
 
+	isFileVolumesWithVmServiceVmSupported := commonco.ContainerOrchestratorUtility.IsFSSEnabled(ctx,
+		common.FileVolumesWithVmService)
+
 	var networkNames []string
 	for _, networkInterface := range virtualMachineInstance.Spec.Network.Interfaces {
-		networkNames = append(networkNames, networkInterface.Network.Name)
+		if !isFileVolumesWithVmServiceVmSupported {
+			networkNames = append(networkNames, networkInterface.Network.Name)
+		} else if networkInterface.Network.Name != "" {
+			networkNames = append(networkNames, networkInterface.Network.Name)
+		}
 	}
 	log.Debugf("VirtualMachine %s/%s is configured with networks %v", vmNamespace, vmName, networkNames)
 
@@ -137,8 +152,27 @@ func GetTKGVMIP(ctx context.Context, vmOperatorClient client.Client, dc dynamic.
 			}
 		}
 		if ip == "" {
-			return "", fmt.Errorf("failed to get SNAT IP annotation from VirtualMachine %s/%s",
-				vmNamespace, vmName)
+			if !isFileVolumesWithVmServiceVmSupported {
+				return "", fmt.Errorf("failed to get SNAT IP annotation from VirtualMachine %s/%s",
+					vmNamespace, vmName)
+			}
+			if len(networkNames) != 0 {
+				// If networkNames for VirtualNetwork were found on the VM,
+				// then some error happened in getting the SNAT IP from VirtualNetwork CR.
+				return "", fmt.Errorf("failed to get SNAT IP annotation for VirtualMachine %s/%s "+
+					"from VirtualNetwrok",
+					vmNamespace, vmName)
+			}
+			// It is likely an NSX setup with VM service VMs.
+			// For TKG service VMs, virtual network CR will always be present.
+			ip, err = getSnatIpFromNamespaceNetworkInfo(ctx, dc, vmNamespace, vmName)
+			if err != nil {
+				log.Errorf("failed to get SNAT IP from NameSpaceNetworkInfo. Err %s", err)
+				return "", fmt.Errorf("failed to get SNAT IP from NameSpaceNetworkInfo %s/%s",
+					vmNamespace, vmName)
+			}
+			log.Infof("Obtained SNAT IP %s from NamespaceNetworkInfo for VirtualMachine %s/%s",
+				ip, vmNamespace, vmName)
 		}
 	} else if network_provider_type == VDSNetworkProvider {
 		ip = virtualMachineInstance.Status.Network.PrimaryIP4
@@ -188,6 +222,35 @@ func GetTKGVMIP(ctx context.Context, vmOperatorClient client.Client, dc dynamic.
 	return ip, nil
 }
 
+// getSnatIpFromNamespaceNetworkInfo finds VM's SNAT IP from the namespace's default NamespaceNetworkInfo CR.
+func getSnatIpFromNamespaceNetworkInfo(ctx context.Context, dc dynamic.Interface,
+	vmNamespace string, vmName string) (string, error) {
+	log := logger.GetLogger(ctx)
+	log.Infof("Determining SNAT IP for VM %s in namespace %s via NamespaceNetworkInfo CR", vmNamespace, vmName)
+
+	namespaceNetworkInfoInstance, err := dc.Resource(namespaceNetworkInfoGVR).Namespace(vmNamespace).Get(ctx,
+		vmNamespace, metav1.GetOptions{})
+	if err != nil {
+		return "", err
+	}
+	log.Debugf("Got namespaceNetworkInfo instance %s/%s", vmNamespace, namespaceNetworkInfoInstance.GetName())
+	snatIP, found, err := unstructured.NestedString(namespaceNetworkInfoInstance.Object, "topology", "defaultEgressIP")
+	if err != nil {
+		return "", fmt.Errorf("failed to get defaultEgressIP from namespaceNetworkInfo %s/%s with error: %v",
+			vmNamespace, vmName, err)
+
+	}
+	if !found {
+		return "", fmt.Errorf("defaultEgressIP is not found on namespaceNetworkInfo %s/%s", vmNamespace, vmName)
+
+	}
+	if snatIP == "" {
+		return "", fmt.Errorf("empty SNAT IP for VM %s on namespaceNetworkInfo instance %s/%s", vmName, vmNamespace,
+			namespaceNetworkInfoInstance.GetName())
+	}
+	return snatIP, nil
+}
+
 // GetNetworkProvider reads the network-config configmap in Supervisor cluster.
 // Returns the network provider as NSXT_CONTAINER_PLUGIN for NSX-T, or
 // VSPHERE_NETWORK for VDS. Otherwise, returns an error, if network provider is
diff --git a/pkg/syncer/cnsoperator/util/util_test.go b/pkg/syncer/cnsoperator/util/util_test.go
@@ -0,0 +1,135 @@
+/*
+Copyright 2025 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/dynamic/fake"
+)
+
+func TestGetSnatIpFromNamespaceNetworkInfo(t *testing.T) {
+	ctx := context.Background()
+	gvr := schema.GroupVersionResource{
+		Group:    "nsx.vmware.com",
+		Version:  "v1alpha1",
+		Resource: "namespacenetworkinfos",
+	}
+
+	namespace := "test-namespace"
+	vmName := "test-vm"
+
+	tests := []struct {
+		name           string
+		initialObjects []runtime.Object
+		expectedIP     string
+		expectError    bool
+	}{
+		{
+			name: "Happy path - SNAT IP present",
+			initialObjects: []runtime.Object{
+				&unstructured.Unstructured{
+					Object: map[string]interface{}{
+						"apiVersion": "nsx.vmware.com/v1alpha1",
+						"kind":       "NamespaceNetworkInfo",
+						"metadata": map[string]interface{}{
+							"name":      namespace,
+							"namespace": namespace,
+						},
+						"topology": map[string]interface{}{
+							"defaultEgressIP": "10.10.10.10",
+						},
+					},
+				},
+			},
+			expectedIP:  "10.10.10.10",
+			expectError: false,
+		},
+		{
+			name:           "Error - resource not found",
+			initialObjects: []runtime.Object{}, // Nothing in fake client
+			expectedIP:     "",
+			expectError:    true,
+		},
+		{
+			name: "Error - defaultEgressIP missing",
+			initialObjects: []runtime.Object{
+				&unstructured.Unstructured{
+					Object: map[string]interface{}{
+						"apiVersion": "nsx.vmware.com/v1alpha1",
+						"kind":       "NamespaceNetworkInfo",
+						"metadata": map[string]interface{}{
+							"name":      namespace,
+							"namespace": namespace,
+						},
+						"spec": map[string]interface{}{
+							"topology": map[string]interface{}{
+								// defaultEgressIP intentionally missing
+							},
+						},
+					},
+				},
+			},
+			expectedIP:  "",
+			expectError: true,
+		},
+		{
+			name: "Error - defaultEgressIP is empty",
+			initialObjects: []runtime.Object{
+				&unstructured.Unstructured{
+					Object: map[string]interface{}{
+						"apiVersion": "nsx.vmware.com/v1alpha1",
+						"kind":       "NamespaceNetworkInfo",
+						"metadata": map[string]interface{}{
+							"name":      namespace,
+							"namespace": namespace,
+						},
+						"spec": map[string]interface{}{
+							"topology": map[string]interface{}{
+								"defaultEgressIP": "",
+							},
+						},
+					},
+				},
+			},
+			expectedIP:  "",
+			expectError: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			scheme := runtime.NewScheme()
+			scheme.AddKnownTypes(gvr.GroupVersion())
+			fakeClient := fake.NewSimpleDynamicClient(scheme, tt.initialObjects...)
+			ip, err := getSnatIpFromNamespaceNetworkInfo(ctx, fakeClient, namespace, vmName)
+
+			if tt.expectError {
+				assert.Error(t, err)
+				assert.Empty(t, ip)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, tt.expectedIP, ip)
+			}
+		})
+	}
+}
