hide change behind a cns flag

ZetaoZhuang · ZetaoZhuang · commit ffdc341167d0 · 2024-10-03T22:31:38.000-07:00
diff --git a/cns/configuration/cns_config.json b/cns/configuration/cns_config.json
@@ -33,5 +33,6 @@
     "MellanoxMonitorIntervalSecs": 30,
     "AZRSettings": {
         "PopulateHomeAzCacheRetryIntervalSecs": 60
-    }
+    },
+    "OverrideApipaGatewayAddress": false
 }
diff --git a/cns/configuration/configuration.go b/cns/configuration/configuration.go
@@ -52,6 +52,8 @@ type CNSConfig struct {
 	WatchPods                   bool `json:"-"`
 	WireserverIP                string
 	GRPCSettings                GRPCSettings
+	// Flag to override Apipa default Gateway address (169.254.128.1) to 169.254.128.2
+	OverrideApipaGatewayAddress bool
 }
 
 type TelemetrySettings struct {
diff --git a/cns/configuration/configuration_test.go b/cns/configuration/configuration_test.go
@@ -86,9 +86,10 @@ func TestReadConfigFromFile(t *testing.T) {
 				AZRSettings: AZRSettings{
 					PopulateHomeAzCacheRetryIntervalSecs: 60,
 				},
-				UseHTTPS:     true,
-				UseMTLS:      true,
-				WireserverIP: "168.63.129.16",
+				UseHTTPS:                    true,
+				UseMTLS:                     true,
+				WireserverIP:                "168.63.129.16",
+				OverrideApipaGatewayAddress: true,
 			},
 			wantErr: false,
 		},
@@ -220,6 +221,7 @@ func TestSetCNSConfigDefaults(t *testing.T) {
 					IPAddress: "localhost",
 					Port:      8080,
 				},
+				OverrideApipaGatewayAddress: false,
 			},
 		},
 		{
@@ -250,6 +252,7 @@ func TestSetCNSConfigDefaults(t *testing.T) {
 					IPAddress: "192.168.1.1",
 					Port:      9090,
 				},
+				OverrideApipaGatewayAddress: true,
 			},
 			want: CNSConfig{
 				ChannelMode: "Other",
@@ -279,6 +282,7 @@ func TestSetCNSConfigDefaults(t *testing.T) {
 					IPAddress: "192.168.1.1",
 					Port:      9090,
 				},
+				OverrideApipaGatewayAddress: true,
 			},
 		},
 	}
diff --git a/cns/configuration/testdata/good.json b/cns/configuration/testdata/good.json
@@ -34,5 +34,6 @@
     "WireserverIP": "168.63.129.16",
     "AZRSettings": {
         "PopulateHomeAzCacheRetryIntervalSecs": 60
-    }
+    },
+    "OverrideApipaGatewayAddress": true
 }
diff --git a/cns/hnsclient/hnsclient_linux.go b/cns/hnsclient/hnsclient_linux.go
@@ -39,7 +39,8 @@ func CreateHostNCApipaEndpoint(
 	localIPConfiguration cns.IPConfiguration,
 	allowNCToHostCommunication bool,
 	allowHostToNCCommunication bool,
-	ncPolicies []cns.NetworkContainerRequestPolicies) (string, error) {
+	ncPolicies []cns.NetworkContainerRequestPolicies,
+	overrideApipaGatewayAddress bool) (string, error) {
 	return "", nil
 }
 
diff --git a/cns/hnsclient/hnsclient_linux_test.go b/cns/hnsclient/hnsclient_linux_test.go
@@ -17,7 +17,7 @@ func TestLinuxHnsNetwork(t *testing.T) {
 	require.Error(t, CreateHnsNetwork(cns.CreateHnsNetworkRequest{}))
 	require.Error(t, DeleteHnsNetwork(""))
 	// these no-op but return no error
-	_, err := CreateHostNCApipaEndpoint("", cns.IPConfiguration{}, false, false, []cns.NetworkContainerRequestPolicies{})
+	_, err := CreateHostNCApipaEndpoint("", cns.IPConfiguration{}, false, false, []cns.NetworkContainerRequestPolicies{}, false)
 	require.NoError(t, err)
 	require.NoError(t, DeleteHostNCApipaEndpoint(""))
 }
diff --git a/cns/hnsclient/hnsclient_windows.go b/cns/hnsclient/hnsclient_windows.go
@@ -542,12 +542,9 @@ func configureHostNCApipaEndpoint(
 }
 
 // CreateHostNCApipaEndpoint creates the endpoint in the apipa network for host container connectivity
-func CreateHostNCApipaEndpoint(
-	networkContainerID string,
-	localIPConfiguration cns.IPConfiguration,
-	allowNCToHostCommunication bool,
-	allowHostToNCCommunication bool,
-	ncPolicies []cns.NetworkContainerRequestPolicies) (string, error) {
+func CreateHostNCApipaEndpoint(networkContainerID string, localIPConfiguration cns.IPConfiguration,
+	allowNCToHostCommunication bool, allowHostToNCCommunication bool, ncPolicies []cns.NetworkContainerRequestPolicies,
+	overrideApipaGatewayAddress bool) (string, error) {
 	var (
 		network      *hcn.HostComputeNetwork
 		endpoint     *hcn.HostComputeEndpoint
@@ -572,7 +569,9 @@ func CreateHostNCApipaEndpoint(
 		return endpoint.Id, nil
 	}
 
-	adhocAdjustIPConfig(&localIPConfiguration)
+	if overrideApipaGatewayAddress {
+		adhocAdjustIPConfig(&localIPConfiguration)
+	}
 	if network, err = createHostNCApipaNetwork(localIPConfiguration); err != nil {
 		logger.Errorf("[Azure CNS] Failed to create HostNCApipaNetwork. Error: %v", err)
 		return "", err
diff --git a/cns/restserver/api.go b/cns/restserver/api.go
@@ -1409,12 +1409,7 @@ func (service *HTTPRestService) CreateHostNCApipaEndpoint(w http.ResponseWriter,
 					"AllowNCToHostCommunication or AllowHostToNCCommunication is set to true")
 				returnCode = types.InvalidRequest
 			} else {
-				if endpointID, err = hnsclient.CreateHostNCApipaEndpoint(
-					req.NetworkContainerID,
-					networkContainerDetails.CreateNetworkContainerRequest.LocalIPConfiguration,
-					networkContainerDetails.CreateNetworkContainerRequest.AllowNCToHostCommunication,
-					networkContainerDetails.CreateNetworkContainerRequest.AllowHostToNCCommunication,
-					networkContainerDetails.CreateNetworkContainerRequest.EndpointPolicies); err != nil {
+				if endpointID, err = hnsclient.CreateHostNCApipaEndpoint(req.NetworkContainerID, networkContainerDetails.CreateNetworkContainerRequest.LocalIPConfiguration, networkContainerDetails.CreateNetworkContainerRequest.AllowNCToHostCommunication, networkContainerDetails.CreateNetworkContainerRequest.AllowHostToNCCommunication, networkContainerDetails.CreateNetworkContainerRequest.EndpointPolicies, service.overrideApipaGatewayAddress); err != nil {
 					returnMessage = fmt.Sprintf("CreateHostNCApipaEndpoint failed with error: %v", err)
 					returnCode = types.UnexpectedError
 				}
diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go
@@ -70,14 +70,15 @@ type HTTPRestService struct {
 	state                    *httpRestServiceState
 	podsPendingIPAssignment  *bounded.TimedSet
 	sync.RWMutex
-	dncPartitionKey            string
-	EndpointState              map[string]*EndpointInfo // key : container id
-	EndpointStateStore         store.KeyValueStore
-	cniConflistGenerator       CNIConflistGenerator
-	generateCNIConflistOnce    sync.Once
-	IPConfigsHandlerMiddleware cns.IPConfigsHandlerMiddleware
-	PnpIDByMacAddress          map[string]string
-	imdsClient                 imdsClient
+	dncPartitionKey             string
+	EndpointState               map[string]*EndpointInfo // key : container id
+	EndpointStateStore          store.KeyValueStore
+	cniConflistGenerator        CNIConflistGenerator
+	generateCNIConflistOnce     sync.Once
+	IPConfigsHandlerMiddleware  cns.IPConfigsHandlerMiddleware
+	PnpIDByMacAddress           map[string]string
+	imdsClient                  imdsClient
+	overrideApipaGatewayAddress bool
 }
 
 type CNIConflistGenerator interface {
@@ -169,6 +170,7 @@ type networkInfo struct {
 func NewHTTPRestService(config *common.ServiceConfig, wscli interfaceGetter, wsproxy wireserverProxy, nmagentClient nmagentClient,
 	endpointStateStore store.KeyValueStore, gen CNIConflistGenerator, homeAzMonitor *HomeAzMonitor,
 	imdsClient imdsClient,
+	overrideApipaGatewayAddress bool,
 ) (*HTTPRestService, error) {
 	service, err := cns.NewService(config.Name, config.Version, config.ChannelMode, config.Store)
 	if err != nil {
@@ -214,24 +216,25 @@ func NewHTTPRestService(config *common.ServiceConfig, wscli interfaceGetter, wsp
 	}
 
 	return &HTTPRestService{
-		Service:                  service,
-		store:                    service.Service.Store,
-		dockerClient:             dc,
-		wscli:                    wscli,
-		ipamClient:               ic,
-		nma:                      nmagentClient,
-		wsproxy:                  wsproxy,
-		networkContainer:         nc,
-		PodIPIDByPodInterfaceKey: podIPIDByPodInterfaceKey,
-		PodIPConfigState:         podIPConfigState,
-		routingTable:             routingTable,
-		state:                    serviceState,
-		podsPendingIPAssignment:  bounded.NewTimedSet(250), // nolint:gomnd // maxpods
-		EndpointStateStore:       endpointStateStore,
-		EndpointState:            make(map[string]*EndpointInfo),
-		homeAzMonitor:            homeAzMonitor,
-		cniConflistGenerator:     gen,
-		imdsClient:               imdsClient,
+		Service:                     service,
+		store:                       service.Service.Store,
+		dockerClient:                dc,
+		wscli:                       wscli,
+		ipamClient:                  ic,
+		nma:                         nmagentClient,
+		wsproxy:                     wsproxy,
+		networkContainer:            nc,
+		PodIPIDByPodInterfaceKey:    podIPIDByPodInterfaceKey,
+		PodIPConfigState:            podIPConfigState,
+		routingTable:                routingTable,
+		state:                       serviceState,
+		podsPendingIPAssignment:     bounded.NewTimedSet(250), // nolint:gomnd // maxpods
+		EndpointStateStore:          endpointStateStore,
+		EndpointState:               make(map[string]*EndpointInfo),
+		homeAzMonitor:               homeAzMonitor,
+		cniConflistGenerator:        gen,
+		imdsClient:                  imdsClient,
+		overrideApipaGatewayAddress: overrideApipaGatewayAddress,
 	}, nil
 }
 
diff --git a/cns/service/main.go b/cns/service/main.go
@@ -751,7 +751,7 @@ func main() {
 	imdsClient := imds.NewClient()
 
 	httpRemoteRestService, err := restserver.NewHTTPRestService(&config, wsclient, &wsProxy, nmaClient,
-		endpointStateStore, conflistGenerator, homeAzMonitor, imdsClient)
+		endpointStateStore, conflistGenerator, homeAzMonitor, imdsClient, cnsconfig.OverrideApipaGatewayAddress)
 	if err != nil {
 		logger.Errorf("Failed to create CNS object, err:%v.\n", err)
 		return

Original file line number	Diff line number	Diff line change
`@@ -33,5 +33,6 @@`
`33`	`33`	`"MellanoxMonitorIntervalSecs": 30,`
`34`	`34`	`"AZRSettings": {`
`35`	`35`	`"PopulateHomeAzCacheRetryIntervalSecs": 60`
`36`		`- }`
	`36`	`+ },`
	`37`	`+ "OverrideApipaGatewayAddress": false`
`37`	`38`	`}`
Original file line number	Diff line number	Diff line change
`@@ -52,6 +52,8 @@ type CNSConfig struct {`
`52`	`52`	WatchPods bool `json:"-"`
`53`	`53`	`WireserverIP string`
`54`	`54`	`GRPCSettings GRPCSettings`
	`55`	`+ // Flag to override Apipa default Gateway address (169.254.128.1) to 169.254.128.2`
	`56`	`+ OverrideApipaGatewayAddress bool`
`55`	`57`	`}`
`56`	`58`
`57`	`59`	`type TelemetrySettings struct {`
Original file line number	Diff line number	Diff line change
`@@ -34,5 +34,6 @@`
`34`	`34`	`"WireserverIP": "168.63.129.16",`
`35`	`35`	`"AZRSettings": {`
`36`	`36`	`"PopulateHomeAzCacheRetryIntervalSecs": 60`
`37`		`- }`
	`37`	`+ },`
	`38`	`+ "OverrideApipaGatewayAddress": true`
`38`	`39`	`}`
Original file line number	Diff line number	Diff line change
`@@ -39,7 +39,8 @@ func CreateHostNCApipaEndpoint(`
`39`	`39`	`localIPConfiguration cns.IPConfiguration,`
`40`	`40`	`allowNCToHostCommunication bool,`
`41`	`41`	`allowHostToNCCommunication bool,`
`42`		`- ncPolicies []cns.NetworkContainerRequestPolicies) (string, error) {`
	`42`	`+ ncPolicies []cns.NetworkContainerRequestPolicies,`
	`43`	`+ overrideApipaGatewayAddress bool) (string, error) {`
`43`	`44`	`return "", nil`
`44`	`45`	`}`
`45`	`46`
Original file line number	Diff line number	Diff line change
`@@ -17,7 +17,7 @@ func TestLinuxHnsNetwork(t *testing.T) {`
`17`	`17`	`require.Error(t, CreateHnsNetwork(cns.CreateHnsNetworkRequest{}))`
`18`	`18`	`require.Error(t, DeleteHnsNetwork(""))`
`19`	`19`	`// these no-op but return no error`
`20`		`- _, err := CreateHostNCApipaEndpoint("", cns.IPConfiguration{}, false, false, []cns.NetworkContainerRequestPolicies{})`
	`20`	`+ _, err := CreateHostNCApipaEndpoint("", cns.IPConfiguration{}, false, false, []cns.NetworkContainerRequestPolicies{}, false)`
`21`	`21`	`require.NoError(t, err)`
`22`	`22`	`require.NoError(t, DeleteHostNCApipaEndpoint(""))`
`23`	`23`	`}`