Skip to content

Commit 1c1bbaa

Browse files
santhoshmprabhutimraymond
santhoshmprabhu
and
timraymond
authored
feat: support for cilium + nodesubnet (#3073)
* feat: support for cilium + nodesubnet * fix: make linter happy * fix: make linter happy * fix: make linter happy * test: add test for nodesubnet * chore: add missing files * nicer comment * chore: fix comment typo * fix: update cns/restserver/nodesubnet.go Co-authored-by: Timothy J. Raymond <[email protected]> Signed-off-by: Santhosh Prabhu <[email protected]> * fix: update cns/restserver/restserver.go Co-authored-by: Timothy J. Raymond <[email protected]> Signed-off-by: Santhosh Prabhu <[email protected]> * refactor: address comments * fix: address comments * chore:comment cleanup * fix: do not use bash in ip config update * fix: address comments * fix: make linter happy * chore: move pipeline changes out * test: more elaborate test including checks on IP pool state * fix: use comments suitable for documentation Co-authored-by: Timothy J. Raymond <[email protected]> Signed-off-by: Santhosh Prabhu <[email protected]> * chore: address comments * chore:make linter happy * fix: address comments * chore: typo * chore: address comments * fix: update comments --------- Signed-off-by: Santhosh Prabhu <[email protected]> Co-authored-by: Timothy J. Raymond <[email protected]>
1 parent 7f0339a commit 1c1bbaa

File tree

6 files changed

+391
-37
lines changed

6 files changed

+391
-37
lines changed

‎cns/fakes/nmagentclientfake.go‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,10 @@ import (
1414

1515
// NMAgentClientFake can be used to query to VM Host info.
1616
type NMAgentClientFake struct {
17-
SupportedAPIsF func(context.Context) ([]string, error)
18-
GetNCVersionListF func(context.Context) (nmagent.NCVersionList, error)
19-
GetHomeAzF func(context.Context) (nmagent.AzResponse, error)
17+
SupportedAPIsF func(context.Context) ([]string, error)
18+
GetNCVersionListF func(context.Context) (nmagent.NCVersionList, error)
19+
GetHomeAzF func(context.Context) (nmagent.AzResponse, error)
20+
GetInterfaceIPInfoF func(ctx context.Context) (nmagent.Interfaces, error)
2021
}
2122

2223
func (n *NMAgentClientFake) SupportedAPIs(ctx context.Context) ([]string, error) {
@@ -30,3 +31,7 @@ func (n *NMAgentClientFake) GetNCVersionList(ctx context.Context) (nmagent.NCVer
3031
func (n *NMAgentClientFake) GetHomeAz(ctx context.Context) (nmagent.AzResponse, error) {
3132
return n.GetHomeAzF(ctx)
3233
}
34+
35+
func (n *NMAgentClientFake) GetInterfaceIPInfo(ctx context.Context) (nmagent.Interfaces, error) {
36+
return n.GetInterfaceIPInfoF(ctx)
37+
}

‎cns/restserver/helper_for_nodesubnet_test.go‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
package restserver
2+
3+
import (
4+
"context"
5+
"net/netip"
6+
"testing"
7+
8+
"github.com/Azure/azure-container-networking/cns"
9+
"github.com/Azure/azure-container-networking/cns/common"
10+
"github.com/Azure/azure-container-networking/cns/fakes"
11+
"github.com/Azure/azure-container-networking/cns/nodesubnet"
12+
acn "github.com/Azure/azure-container-networking/common"
13+
"github.com/Azure/azure-container-networking/nmagent"
14+
"github.com/Azure/azure-container-networking/store"
15+
)
16+
17+
// GetRestServiceObjectForNodeSubnetTest creates a new HTTPRestService object for use in nodesubnet unit tests.
18+
func GetRestServiceObjectForNodeSubnetTest(t *testing.T, generator CNIConflistGenerator) *HTTPRestService {
19+
config := &common.ServiceConfig{
20+
Name: "test",
21+
Version: "1.0",
22+
ChannelMode: "AzureHost",
23+
Store: store.NewMockStore("test"),
24+
}
25+
interfaces := nmagent.Interfaces{
26+
Entries: []nmagent.Interface{
27+
{
28+
MacAddress: nmagent.MACAddress{0x00, 0x0D, 0x3A, 0xF9, 0xDC, 0xA6},
29+
IsPrimary: true,
30+
InterfaceSubnets: []nmagent.InterfaceSubnet{
31+
{
32+
Prefix: "10.0.0.0/24",
33+
IPAddress: []nmagent.NodeIP{
34+
{
35+
Address: nmagent.IPAddress(netip.AddrFrom4([4]byte{10, 0, 0, 4})),
36+
IsPrimary: true,
37+
},
38+
{
39+
Address: nmagent.IPAddress(netip.AddrFrom4([4]byte{10, 0, 0, 52})),
40+
IsPrimary: false,
41+
},
42+
{
43+
Address: nmagent.IPAddress(netip.AddrFrom4([4]byte{10, 0, 0, 63})),
44+
IsPrimary: false,
45+
},
46+
{
47+
Address: nmagent.IPAddress(netip.AddrFrom4([4]byte{10, 0, 0, 45})),
48+
IsPrimary: false,
49+
},
50+
},
51+
},
52+
},
53+
},
54+
},
55+
}
56+
57+
svc, err := cns.NewService(config.Name, config.Version, config.ChannelMode, config.Store)
58+
if err != nil {
59+
return nil
60+
}
61+
62+
svc.SetOption(acn.OptCnsURL, "")
63+
svc.SetOption(acn.OptCnsPort, "")
64+
err = svc.Initialize(config)
65+
if err != nil {
66+
return nil
67+
}
68+
69+
t.Cleanup(func() { svc.Uninitialize() })
70+
71+
return &HTTPRestService{
72+
Service: svc,
73+
cniConflistGenerator: generator,
74+
state: &httpRestServiceState{},
75+
PodIPConfigState: make(map[string]cns.IPConfigurationStatus),
76+
PodIPIDByPodInterfaceKey: make(map[string][]string),
77+
nma: &fakes.NMAgentClientFake{
78+
GetInterfaceIPInfoF: func(_ context.Context) (nmagent.Interfaces, error) {
79+
return interfaces, nil
80+
},
81+
},
82+
wscli: &fakes.WireserverClientFake{},
83+
}
84+
}
85+
86+
// GetNodesubnetIPFetcher gets the nodesubnetIPFetcher from the HTTPRestService.
87+
func (service *HTTPRestService) GetNodesubnetIPFetcher() *nodesubnet.IPFetcher {
88+
return service.nodesubnetIPFetcher
89+
}

‎cns/restserver/nodesubnet.go‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
package restserver
2+
3+
import (
4+
"context"
5+
"net/netip"
6+
7+
"github.com/Azure/azure-container-networking/cns"
8+
"github.com/Azure/azure-container-networking/cns/logger"
9+
nodesubnet "github.com/Azure/azure-container-networking/cns/nodesubnet"
10+
"github.com/Azure/azure-container-networking/cns/types"
11+
"github.com/pkg/errors"
12+
)
13+
14+
var _ nodesubnet.IPConsumer = &HTTPRestService{}
15+
16+
// UpdateIPsForNodeSubnet updates the IP pool of HTTPRestService with newly fetched secondary IPs
17+
func (service *HTTPRestService) UpdateIPsForNodeSubnet(secondaryIPs []netip.Addr) error {
18+
secondaryIPStrs := make([]string, len(secondaryIPs))
19+
for i, ip := range secondaryIPs {
20+
secondaryIPStrs[i] = ip.String()
21+
}
22+
23+
networkContainerRequest := nodesubnet.CreateNodeSubnetNCRequest(secondaryIPStrs)
24+
25+
code, msg := service.saveNetworkContainerGoalState(*networkContainerRequest)
26+
if code != types.Success {
27+
return errors.Errorf("failed to save fetched ips. code: %d, message %s", code, msg)
28+
}
29+
30+
logger.Debugf("IP change processed successfully")
31+
32+
// saved NC successfully. UpdateIPsForNodeSubnet is called only when IPs are fetched from NMAgent.
33+
// We now have IPs to serve IPAM requests. Generate conflist to indicate CNS is ready
34+
service.MustGenerateCNIConflistOnce()
35+
return nil
36+
}
37+
38+
// InitializeNodeSubnet prepares CNS for serving NodeSubnet requests.
39+
// It sets the orchestrator type to KubernetesCRD, reconciles the initial
40+
// CNS state from the statefile, then creates an IP fetcher.
41+
func (service *HTTPRestService) InitializeNodeSubnet(ctx context.Context, podInfoByIPProvider cns.PodInfoByIPProvider) error {
42+
// set orchestrator type
43+
orchestrator := cns.SetOrchestratorTypeRequest{
44+
OrchestratorType: cns.KubernetesCRD,
45+
}
46+
service.SetNodeOrchestrator(&orchestrator)
47+
48+
if podInfoByIPProvider == nil {
49+
logger.Printf("PodInfoByIPProvider is nil, this usually means no saved endpoint state. Skipping reconciliation")
50+
} else if _, err := nodesubnet.ReconcileInitialCNSState(ctx, service, podInfoByIPProvider); err != nil {
51+
return errors.Wrap(err, "reconcile initial CNS state")
52+
}
53+
// statefile (if any) is reconciled. Initialize the IP fetcher. Start the IP fetcher only after the service is started,
54+
// because starting the IP fetcher will generate conflist, which should be done only once we are ready to respond to IPAM requests.
55+
service.nodesubnetIPFetcher = nodesubnet.NewIPFetcher(service.nma, service, 0, 0, logger.Log)
56+
57+
return nil
58+
}
59+
60+
// StartNodeSubnet starts the IP fetcher for NodeSubnet. This will cause secondary IPs to be fetched periodically.
61+
// After the first successful fetch, conflist will be generated to indicate CNS is ready.
62+
func (service *HTTPRestService) StartNodeSubnet(ctx context.Context) {
63+
service.nodesubnetIPFetcher.Start(ctx)
64+
}

‎cns/restserver/nodesubnet_test.go‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
package restserver_test
2+
3+
import (
4+
"context"
5+
"net"
6+
"testing"
7+
8+
"github.com/Azure/azure-container-networking/cns/cnireconciler"
9+
"github.com/Azure/azure-container-networking/cns/logger"
10+
"github.com/Azure/azure-container-networking/cns/restserver"
11+
"github.com/Azure/azure-container-networking/cns/types"
12+
"github.com/Azure/azure-container-networking/store"
13+
)
14+
15+
// getMockStore creates a mock KeyValueStore with some endpoint state
16+
func getMockStore() store.KeyValueStore {
17+
mockStore := store.NewMockStore("")
18+
endpointState := map[string]*restserver.EndpointInfo{
19+
"12e65d89e58cb23c784e97840cf76866bfc9902089bdc8e87e9f64032e312b0b": {
20+
PodName: "coredns-54b69f46b8-ldmwr",
21+
PodNamespace: "kube-system",
22+
IfnameToIPMap: map[string]*restserver.IPInfo{
23+
"eth0": {
24+
IPv4: []net.IPNet{
25+
{
26+
IP: net.IPv4(10, 0, 0, 52),
27+
Mask: net.CIDRMask(24, 32),
28+
},
29+
},
30+
},
31+
},
32+
},
33+
"1fc5176913a3a1a7facfb823dde3b4ded404041134fef4f4a0c8bba140fc0413": {
34+
PodName: "load-test-7f7d49687d-wxc9p",
35+
PodNamespace: "load-test",
36+
IfnameToIPMap: map[string]*restserver.IPInfo{
37+
"eth0": {
38+
IPv4: []net.IPNet{
39+
{
40+
IP: net.IPv4(10, 0, 0, 63),
41+
Mask: net.CIDRMask(24, 32),
42+
},
43+
},
44+
},
45+
},
46+
},
47+
}
48+
49+
err := mockStore.Write(restserver.EndpointStoreKey, endpointState)
50+
if err != nil {
51+
return nil
52+
}
53+
return mockStore
54+
}
55+
56+
// Mock implementation of CNIConflistGenerator
57+
type MockCNIConflistGenerator struct {
58+
GenerateCalled chan struct{}
59+
}
60+
61+
func (m *MockCNIConflistGenerator) Generate() error {
62+
close(m.GenerateCalled)
63+
return nil
64+
}
65+
66+
func (m *MockCNIConflistGenerator) Close() error {
67+
// Implement the Close method logic here if needed
68+
return nil
69+
}
70+
71+
// TestNodeSubnet tests initialization of NodeSubnet with endpoint info, and verfies that
72+
// the conflist is generated after fetching secondary IPs
73+
func TestNodeSubnet(t *testing.T) {
74+
podInfoByIPProvider, err := cnireconciler.NewCNSPodInfoProvider(getMockStore())
75+
if err != nil {
76+
t.Fatalf("NewCNSPodInfoProvider returned an error: %v", err)
77+
}
78+
79+
// create a real HTTPRestService object
80+
mockCNIConflistGenerator := &MockCNIConflistGenerator{
81+
GenerateCalled: make(chan struct{}),
82+
}
83+
service := restserver.GetRestServiceObjectForNodeSubnetTest(t, mockCNIConflistGenerator)
84+
ctx, cancel := testContext(t)
85+
defer cancel()
86+
87+
err = service.InitializeNodeSubnet(ctx, podInfoByIPProvider)
88+
if err != nil {
89+
t.Fatalf("InitializeNodeSubnet returned an error: %v", err)
90+
}
91+
92+
expectedIPs := map[string]types.IPState{
93+
"10.0.0.52": types.Assigned,
94+
"10.0.0.63": types.Assigned,
95+
}
96+
97+
checkIPassignment(t, service, expectedIPs)
98+
99+
service.StartNodeSubnet(ctx)
100+
101+
if service.GetNodesubnetIPFetcher() == nil {
102+
t.Fatal("NodeSubnetIPFetcher is not initialized")
103+
}
104+
105+
select {
106+
case <-ctx.Done():
107+
t.Errorf("test context done - %s", ctx.Err())
108+
return
109+
case <-mockCNIConflistGenerator.GenerateCalled:
110+
break
111+
}
112+
113+
expectedIPs["10.0.0.45"] = types.Available
114+
checkIPassignment(t, service, expectedIPs)
115+
}
116+
117+
// checkIPassignment checks whether the IP assignment state in the HTTPRestService object matches expectation
118+
func checkIPassignment(t *testing.T, service *restserver.HTTPRestService, expectedIPs map[string]types.IPState) {
119+
if len(service.PodIPConfigState) != len(expectedIPs) {
120+
t.Fatalf("expected 2 entries in PodIPConfigState, got %d", len(service.PodIPConfigState))
121+
}
122+
123+
for ip := range service.GetPodIPConfigState() {
124+
config := service.GetPodIPConfigState()[ip]
125+
if assignmentState, exists := expectedIPs[ip]; !exists {
126+
t.Fatalf("unexpected IP %s in PodIPConfigState", ip)
127+
} else if config.GetState() != assignmentState {
128+
t.Fatalf("expected state 'Assigned' for IP %s, got %s", ip, config.GetState())
129+
}
130+
}
131+
}
132+
133+
// testContext creates a context from the provided testing.T that will be
134+
// canceled if the test suite is terminated.
135+
func testContext(t *testing.T) (context.Context, context.CancelFunc) {
136+
if deadline, ok := t.Deadline(); ok {
137+
return context.WithDeadline(context.Background(), deadline)
138+
}
139+
return context.WithCancel(context.Background())
140+
}
141+
142+
func init() {
143+
logger.InitLogger("testlogs", 0, 0, "./")
144+
}

‎cns/restserver/restserver.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"github.com/Azure/azure-container-networking/cns/dockerclient"
1414
"github.com/Azure/azure-container-networking/cns/logger"
1515
"github.com/Azure/azure-container-networking/cns/networkcontainers"
16+
"github.com/Azure/azure-container-networking/cns/nodesubnet"
1617
"github.com/Azure/azure-container-networking/cns/routes"
1718
"github.com/Azure/azure-container-networking/cns/types"
1819
"github.com/Azure/azure-container-networking/cns/types/bounded"
@@ -40,6 +41,7 @@ type nmagentClient interface {
4041
SupportedAPIs(context.Context) ([]string, error)
4142
GetNCVersionList(context.Context) (nma.NCVersionList, error)
4243
GetHomeAz(context.Context) (nma.AzResponse, error)
44+
GetInterfaceIPInfo(ctx context.Context) (nma.Interfaces, error)
4345
}
4446

4547
type wireserverProxy interface {
@@ -76,6 +78,7 @@ type HTTPRestService struct {
7678
IPConfigsHandlerMiddleware cns.IPConfigsHandlerMiddleware
7779
PnpIDByMacAddress map[string]string
7880
imdsClient imdsClient
81+
nodesubnetIPFetcher *nodesubnet.IPFetcher
7982
}
8083

8184
type CNIConflistGenerator interface {

0 commit comments

Comments
 (0)