Skip to content

Commit 455f5cb

Browse files
vakalapavakalapa
authored
[NPM] Decoupling resource cache maps from NSmap (#820)
* first pass at decoupling resource maps * First pass on decoupling resource maps * Adding telemetry capabilities to resource CRUD events * Initializing new maps in nprMgr for tests * Initializing new maps in nprMgr for tests * Adding artifact for Npm logs * Addressing comments * Addressing comments
1 parent 4096ab1 commit 455f5cb

File tree

11 files changed

+348
-165
lines changed

11 files changed

+348
-165
lines changed

‎.pipelines/npm/npm-conformance-tests.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,10 @@ jobs:
179179
mkdir -p $(System.DefaultWorkingDirectory)/npmLogs
180180
for npm in $npmPodList; do kubectl logs -n kube-system $npm > $(System.DefaultWorkingDirectory)/npmLogs/$npm ;done
181181
displayName: "Gather NPM Logs"
182+
condition: always()
182183
- publish: $(System.DefaultWorkingDirectory)/npmLogs
184+
condition: always()
185+
artifact: NpmLogs
183186

184187
- job: Clean_up
185188
displayName: "Cleanup"

‎npm/http/server/server_test.go‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,9 @@ import (
1515
func TestGetNpmMgrHandler(t *testing.T) {
1616
assert := assert.New(t)
1717
npMgr := &npm.NetworkPolicyManager{
18-
NsMap: map[string]*npm.Namespace{
19-
"test": &npm.Namespace{
20-
PodMap: map[string]*npm.NpmPod{
21-
"": &npm.NpmPod{
22-
Name: "testpod",
23-
},
24-
},
18+
PodMap: map[string]*npm.NpmPod{
19+
"": &npm.NpmPod{
20+
Name: "testpod",
2521
},
2622
},
2723
}

‎npm/namespace.go‎

Lines changed: 21 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -8,19 +8,16 @@ import (
88
"github.com/Azure/azure-container-networking/log"
99
"github.com/Azure/azure-container-networking/npm/ipsm"
1010
"github.com/Azure/azure-container-networking/npm/iptm"
11+
"github.com/Azure/azure-container-networking/npm/metrics"
1112
"github.com/Azure/azure-container-networking/npm/util"
1213

1314
corev1 "k8s.io/api/core/v1"
14-
networkingv1 "k8s.io/api/networking/v1"
1515
)
1616

1717
type Namespace struct {
1818
name string
1919
LabelsMap map[string]string // NameSpace labels
2020
SetMap map[string]string
21-
PodMap map[string]*NpmPod // Key is PodUID
22-
rawNpMap map[string]*networkingv1.NetworkPolicy
23-
ProcessedNpMap map[string]*networkingv1.NetworkPolicy
2421
IpsMgr *ipsm.IpsetManager
2522
iptMgr *iptm.IptablesManager
2623
resourceVersion uint64 // NameSpace ResourceVersion
@@ -29,14 +26,11 @@ type Namespace struct {
2926
// newNS constructs a new namespace object.
3027
func newNs(name string) (*Namespace, error) {
3128
ns := &Namespace{
32-
name: name,
33-
LabelsMap: make(map[string]string),
34-
SetMap: make(map[string]string),
35-
PodMap: make(map[string]*NpmPod),
36-
rawNpMap: make(map[string]*networkingv1.NetworkPolicy),
37-
ProcessedNpMap: make(map[string]*networkingv1.NetworkPolicy),
38-
IpsMgr: ipsm.NewIpsetManager(),
39-
iptMgr: iptm.NewIptablesManager(),
29+
name: name,
30+
LabelsMap: make(map[string]string),
31+
SetMap: make(map[string]string),
32+
IpsMgr: ipsm.NewIpsetManager(),
33+
iptMgr: iptm.NewIptablesManager(),
4034
// resource version is converted to uint64
4135
// so make sure it is initialized to "0"
4236
resourceVersion: 0,
@@ -63,28 +57,6 @@ func isInvalidNamespaceUpdate(oldNsObj, newNsObj *corev1.Namespace) (isInvalidUp
6357
return
6458
}
6559

66-
func (ns *Namespace) policyExists(npObj *networkingv1.NetworkPolicy) bool {
67-
np, exists := ns.rawNpMap[npObj.ObjectMeta.Name]
68-
if !exists {
69-
return false
70-
}
71-
72-
if !util.CompareResourceVersions(np.ObjectMeta.ResourceVersion, npObj.ObjectMeta.ResourceVersion) {
73-
log.Logf("Cached Network Policy has larger ResourceVersion number than new Obj. Name: %s Cached RV: %d New RV: %d\n",
74-
npObj.ObjectMeta.Name,
75-
np.ObjectMeta.ResourceVersion,
76-
npObj.ObjectMeta.ResourceVersion,
77-
)
78-
return true
79-
}
80-
81-
if isSamePolicy(np, npObj) {
82-
return true
83-
}
84-
85-
return false
86-
}
87-
8860
// InitAllNsList syncs all-namespace ipset list.
8961
func (npMgr *NetworkPolicyManager) InitAllNsList() error {
9062
allNs := npMgr.NsMap[util.KubeAllNamespacesFlag]
@@ -94,7 +66,7 @@ func (npMgr *NetworkPolicyManager) InitAllNsList() error {
9466
}
9567

9668
if err := allNs.IpsMgr.AddToList(util.KubeAllNamespacesFlag, ns); err != nil {
97-
log.Errorf("Error: failed to add namespace set %s to ipset list %s", ns, util.KubeAllNamespacesFlag)
69+
metrics.SendErrorLogAndMetric(util.NSID, "[InitAllNsList] Error: failed to add namespace set %s to ipset list %s with err: %v", ns, util.KubeAllNamespacesFlag, err)
9870
return err
9971
}
10072
}
@@ -111,7 +83,7 @@ func (npMgr *NetworkPolicyManager) UninitAllNsList() error {
11183
}
11284

11385
if err := allNs.IpsMgr.DeleteFromList(util.KubeAllNamespacesFlag, ns); err != nil {
114-
log.Errorf("Error: failed to delete namespace set %s from list %s", ns, util.KubeAllNamespacesFlag)
86+
metrics.SendErrorLogAndMetric(util.NSID, "[UninitAllNsList] Error: failed to delete namespace set %s from list %s with err: %v", ns, util.KubeAllNamespacesFlag, err)
11587
return err
11688
}
11789
}
@@ -129,12 +101,12 @@ func (npMgr *NetworkPolicyManager) AddNamespace(nsObj *corev1.Namespace) error {
129101
ipsMgr := npMgr.NsMap[util.KubeAllNamespacesFlag].IpsMgr
130102
// Create ipset for the namespace.
131103
if err = ipsMgr.CreateSet(nsName, append([]string{util.IpsetNetHashFlag})); err != nil {
132-
log.Errorf("Error: failed to create ipset for namespace %s.", nsName)
104+
metrics.SendErrorLogAndMetric(util.NSID, "[AddNamespace] Error: failed to create ipset for namespace %s with err: %v", nsName, err)
133105
return err
134106
}
135107

136108
if err = ipsMgr.AddToList(util.KubeAllNamespacesFlag, nsName); err != nil {
137-
log.Errorf("Error: failed to add %s to all-namespace ipset list.", nsName)
109+
metrics.SendErrorLogAndMetric(util.NSID, "[AddNamespace] Error: failed to add %s to all-namespace ipset list with err: %v", nsName, err)
138110
return err
139111
}
140112

@@ -144,21 +116,21 @@ func (npMgr *NetworkPolicyManager) AddNamespace(nsObj *corev1.Namespace) error {
144116
labelKey := util.GetNSNameWithPrefix(nsLabelKey)
145117
log.Logf("Adding namespace %s to ipset list %s", nsName, labelKey)
146118
if err = ipsMgr.AddToList(labelKey, nsName); err != nil {
147-
log.Errorf("Error: failed to add namespace %s to ipset list %s", nsName, labelKey)
119+
metrics.SendErrorLogAndMetric(util.NSID, "[AddNamespace] Error: failed to add namespace %s to ipset list %s with err: %v", nsName, labelKey, err)
148120
return err
149121
}
150122

151123
label := util.GetNSNameWithPrefix(nsLabelKey + ":" + nsLabelVal)
152124
log.Logf("Adding namespace %s to ipset list %s", nsName, label)
153125
if err = ipsMgr.AddToList(label, nsName); err != nil {
154-
log.Errorf("Error: failed to add namespace %s to ipset list %s", nsName, label)
126+
metrics.SendErrorLogAndMetric(util.NSID, "[AddNamespace] Error: failed to add namespace %s to ipset list %s with err: %v", nsName, label, err)
155127
return err
156128
}
157129
}
158130

159131
ns, err := newNs(nsName)
160132
if err != nil {
161-
log.Errorf("Error: failed to create namespace %s", nsName)
133+
metrics.SendErrorLogAndMetric(util.NSID, "[AddNamespace] Error: failed to create namespace %s with err: %v", nsName, err)
162134
}
163135
setResourceVersion(ns, nsObj.GetObjectMeta().GetResourceVersion())
164136

@@ -223,8 +195,8 @@ func (npMgr *NetworkPolicyManager) UpdateNamespace(oldNsObj *corev1.Namespace, n
223195
//if no change in labels then return
224196
if reflect.DeepEqual(curNsObj.LabelsMap, newNsLabel) {
225197
log.Logf(
226-
"NAMESPACE UPDATING:\n nothing to delete or add. old namespace: [%s/%v]\n cache namespace: [%s/%v] new namespace: [%s/%v]",
227-
oldNsNs, oldNsLabel, curNsObj.name, curNsObj.LabelsMap, newNsNs, newNsLabel,
198+
"NAMESPACE UPDATING: nothing to delete or add. namespace: [%s/%v]",
199+
newNsNs, newNsLabel,
228200
)
229201
return nil
230202
}
@@ -238,7 +210,7 @@ func (npMgr *NetworkPolicyManager) UpdateNamespace(oldNsObj *corev1.Namespace, n
238210
labelKey := util.GetNSNameWithPrefix(nsLabelVal)
239211
log.Logf("Deleting namespace %s from ipset list %s", oldNsNs, labelKey)
240212
if err = ipsMgr.DeleteFromList(labelKey, oldNsNs); err != nil {
241-
log.Errorf("Error: failed to delete namespace %s from ipset list %s", oldNsNs, labelKey)
213+
metrics.SendErrorLogAndMetric(util.NSID, "[UpdateNamespace] Error: failed to delete namespace %s from ipset list %s with err: %v", oldNsNs, labelKey, err)
242214
return err
243215
}
244216
}
@@ -248,7 +220,7 @@ func (npMgr *NetworkPolicyManager) UpdateNamespace(oldNsObj *corev1.Namespace, n
248220
labelKey := util.GetNSNameWithPrefix(nsLabelVal)
249221
log.Logf("Adding namespace %s to ipset list %s", oldNsNs, labelKey)
250222
if err = ipsMgr.AddToList(labelKey, oldNsNs); err != nil {
251-
log.Errorf("Error: failed to add namespace %s to ipset list %s", oldNsNs, labelKey)
223+
metrics.SendErrorLogAndMetric(util.NSID, "[UpdateNamespace] Error: failed to add namespace %s to ipset list %s with err: %v", oldNsNs, labelKey, err)
252224
return err
253225
}
254226
}
@@ -281,27 +253,27 @@ func (npMgr *NetworkPolicyManager) DeleteNamespace(nsObj *corev1.Namespace) erro
281253
labelKey := util.GetNSNameWithPrefix(nsLabelKey)
282254
log.Logf("Deleting namespace %s from ipset list %s", nsName, labelKey)
283255
if err = ipsMgr.DeleteFromList(labelKey, nsName); err != nil {
284-
log.Errorf("Error: failed to delete namespace %s from ipset list %s", nsName, labelKey)
256+
metrics.SendErrorLogAndMetric(util.NSID, "[DeleteNamespace] Error: failed to delete namespace %s from ipset list %s with err: %v", nsName, labelKey, err)
285257
return err
286258
}
287259

288260
label := util.GetNSNameWithPrefix(nsLabelKey + ":" + nsLabelVal)
289261
log.Logf("Deleting namespace %s from ipset list %s", nsName, label)
290262
if err = ipsMgr.DeleteFromList(label, nsName); err != nil {
291-
log.Errorf("Error: failed to delete namespace %s from ipset list %s", nsName, label)
263+
metrics.SendErrorLogAndMetric(util.NSID, "[DeleteNamespace] Error: failed to delete namespace %s from ipset list %s with err: %v", nsName, label, err)
292264
return err
293265
}
294266
}
295267

296268
// Delete the namespace from all-namespace ipset list.
297269
if err = ipsMgr.DeleteFromList(util.KubeAllNamespacesFlag, nsName); err != nil {
298-
log.Errorf("Error: failed to delete namespace %s from ipset list %s", nsName, util.KubeAllNamespacesFlag)
270+
metrics.SendErrorLogAndMetric(util.NSID, "[DeleteNamespace] Error: failed to delete namespace %s from ipset list %s with err: %v", nsName, util.KubeAllNamespacesFlag, err)
299271
return err
300272
}
301273

302274
// Delete ipset for the namespace.
303275
if err = ipsMgr.DeleteSet(nsName); err != nil {
304-
log.Errorf("Error: failed to delete ipset for namespace %s.", nsName)
276+
metrics.SendErrorLogAndMetric(util.NSID, "[DeleteNamespace] Error: failed to delete ipset for namespace %s with err: %v", nsName, err)
305277
return err
306278
}
307279

‎npm/namespace_test.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"github.com/Azure/azure-container-networking/npm/ipsm"
1414
"github.com/Azure/azure-container-networking/npm/util"
1515
corev1 "k8s.io/api/core/v1"
16+
networkingv1 "k8s.io/api/networking/v1"
1617
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1718
)
1819

@@ -48,6 +49,9 @@ func TestAllNsList(t *testing.T) {
4849
func TestAddNamespace(t *testing.T) {
4950
npMgr := &NetworkPolicyManager{
5051
NsMap: make(map[string]*Namespace),
52+
PodMap: make(map[string]*NpmPod),
53+
RawNpMap: make(map[string]*networkingv1.NetworkPolicy),
54+
ProcessedNpMap: make(map[string]*networkingv1.NetworkPolicy),
5155
TelemetryEnabled: false,
5256
}
5357

@@ -87,6 +91,9 @@ func TestAddNamespace(t *testing.T) {
8791
func TestUpdateNamespace(t *testing.T) {
8892
npMgr := &NetworkPolicyManager{
8993
NsMap: make(map[string]*Namespace),
94+
PodMap: make(map[string]*NpmPod),
95+
RawNpMap: make(map[string]*networkingv1.NetworkPolicy),
96+
ProcessedNpMap: make(map[string]*networkingv1.NetworkPolicy),
9097
TelemetryEnabled: false,
9198
}
9299

@@ -139,6 +146,9 @@ func TestUpdateNamespace(t *testing.T) {
139146
func TestAddNamespaceLabel(t *testing.T) {
140147
npMgr := &NetworkPolicyManager{
141148
NsMap: make(map[string]*Namespace),
149+
PodMap: make(map[string]*NpmPod),
150+
RawNpMap: make(map[string]*networkingv1.NetworkPolicy),
151+
ProcessedNpMap: make(map[string]*networkingv1.NetworkPolicy),
142152
TelemetryEnabled: false,
143153
}
144154

@@ -200,6 +210,9 @@ func TestAddNamespaceLabel(t *testing.T) {
200210
func TestDeleteandUpdateNamespaceLabel(t *testing.T) {
201211
npMgr := &NetworkPolicyManager{
202212
NsMap: make(map[string]*Namespace),
213+
PodMap: make(map[string]*NpmPod),
214+
RawNpMap: make(map[string]*networkingv1.NetworkPolicy),
215+
ProcessedNpMap: make(map[string]*networkingv1.NetworkPolicy),
203216
TelemetryEnabled: false,
204217
}
205218

@@ -261,6 +274,9 @@ func TestDeleteandUpdateNamespaceLabel(t *testing.T) {
261274
func TestDeleteNamespace(t *testing.T) {
262275
npMgr := &NetworkPolicyManager{
263276
NsMap: make(map[string]*Namespace),
277+
PodMap: make(map[string]*NpmPod),
278+
RawNpMap: make(map[string]*networkingv1.NetworkPolicy),
279+
ProcessedNpMap: make(map[string]*networkingv1.NetworkPolicy),
264280
TelemetryEnabled: false,
265281
}
266282

‎npm/npm.go‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,9 @@ type NetworkPolicyManager struct {
5050

5151
NodeName string
5252
NsMap map[string]*Namespace
53+
PodMap map[string]*NpmPod // Key is ns-<nsname>/<podname>/<poduuid>
54+
RawNpMap map[string]*networkingv1.NetworkPolicy // Key is ns-<nsname>/<policyname>
55+
ProcessedNpMap map[string]*networkingv1.NetworkPolicy // Key is ns-<nsname>/<podSelectorHash>
5356
isAzureNpmChainCreated bool
5457
isSafeToCleanUpAzureNpmChain bool
5558

@@ -120,10 +123,8 @@ func (npMgr *NetworkPolicyManager) SendClusterMetrics() {
120123
podCount.Value = 0
121124
//Reducing one to remove all-namespaces ns obj
122125
nsCount.Value = float64(len(npMgr.NsMap) - 1)
123-
for _, ns := range npMgr.NsMap {
124-
nwPolicyCount.Value += float64(len(ns.rawNpMap))
125-
podCount.Value += float64(len(ns.PodMap))
126-
}
126+
nwPolicyCount.Value += float64(len(npMgr.RawNpMap))
127+
podCount.Value += float64(len(npMgr.PodMap))
127128
npMgr.Unlock()
128129

129130
metrics.SendMetric(podCount)
@@ -232,6 +233,9 @@ func NewNetworkPolicyManager(clientset *kubernetes.Clientset, informerFactory in
232233
npInformer: npInformer,
233234
NodeName: os.Getenv("HOSTNAME"),
234235
NsMap: make(map[string]*Namespace),
236+
PodMap: make(map[string]*NpmPod),
237+
RawNpMap: make(map[string]*networkingv1.NetworkPolicy),
238+
ProcessedNpMap: make(map[string]*networkingv1.NetworkPolicy),
235239
isAzureNpmChainCreated: false,
236240
isSafeToCleanUpAzureNpmChain: false,
237241
clusterState: telemetry.ClusterState{

0 commit comments

Comments
 (0)