Skip to content

Commit 36b5907

Browse files
author
David Wertenteil
authored
Merge pull request kubescape#107 from kubescape/all-lists-refactor
Refactor listing.go package for memory optimizations
2 parents ca6a09d + 5f7866f commit 36b5907

19 files changed

+407
-380
lines changed

go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,9 @@ require (
1010
github.com/kubescape/rbac-utils v0.0.20
1111
github.com/mitchellh/mapstructure v1.1.2
1212
github.com/open-policy-agent/opa v0.42.0
13-
github.com/stretchr/testify v1.8.0
13+
github.com/stretchr/testify v1.8.3
1414
go.uber.org/zap v1.22.0
15+
golang.org/x/exp v0.0.0-20230519143937-03e91628a987
1516
k8s.io/api v0.25.3
1617
k8s.io/apimachinery v0.25.3
1718
k8s.io/client-go v0.25.3
@@ -101,7 +102,6 @@ require (
101102
go.uber.org/atomic v1.7.0 // indirect
102103
go.uber.org/multierr v1.6.0 // indirect
103104
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
104-
golang.org/x/exp v0.0.0-20230116083435-1de6713980de // indirect
105105
golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10 // indirect
106106
golang.org/x/oauth2 v0.0.0-20220630143837-2104d58473e0 // indirect
107107
golang.org/x/sys v0.3.0 // indirect

go.sum

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1086,8 +1086,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
10861086
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
10871087
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
10881088
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
1089-
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
10901089
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
1090+
github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
1091+
github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
10911092
github.com/stripe/stripe-go/v74 v74.8.0 h1:0+3EfQSBhMg8SQ1+w+AP6Gxyko2crWbUG2uXbzYs8SU=
10921093
github.com/stripe/stripe-go/v74 v74.8.0/go.mod h1:5PoXNp30AJ3tGq57ZcFuaMylzNi8KpwlrYAFmO1fHZw=
10931094
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
@@ -1242,8 +1243,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
12421243
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
12431244
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
12441245
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
1245-
golang.org/x/exp v0.0.0-20230116083435-1de6713980de h1:DBWn//IJw30uYCgERoxCg84hWtA97F4wMiKOIh00Uf0=
1246-
golang.org/x/exp v0.0.0-20230116083435-1de6713980de/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
1246+
golang.org/x/exp v0.0.0-20230519143937-03e91628a987 h1:3xJIFvzUFbu4ls0BTBYcgbCGhA63eAOEMxIHugyXJqA=
1247+
golang.org/x/exp v0.0.0-20230519143937-03e91628a987/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
12471248
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
12481249
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
12491250
golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
Lines changed: 116 additions & 107 deletions
Original file line numberDiff line numberDiff line change
@@ -1,145 +1,154 @@
11
package helpers
22

33
import (
4+
"sync"
5+
46
"github.com/kubescape/opa-utils/reporthandling/apis"
5-
"github.com/kubescape/opa-utils/reporthandling/internal/slices"
7+
"golang.org/x/exp/maps"
68
)
79

10+
var allListsPool = &sync.Pool{
11+
New: func() interface{} {
12+
return &AllLists{}
13+
},
14+
}
15+
16+
// GetAllListsFromPool get the AllLists object from the pool
17+
func GetAllListsFromPool() *AllLists {
18+
l := allListsPool.Get().(*AllLists)
19+
// reset the object before returning it as it might be dirty
20+
l.Clear()
21+
return l
22+
}
23+
24+
// PutAllListsToPool put the AllLists object back to the pool
25+
func PutAllListsToPool(l *AllLists) {
26+
allListsPool.Put(l)
27+
}
28+
829
// ReportObject any report object must be compliment with a map[string]interface{} structures
930
type ReportObject map[string]interface{}
1031

1132
// AllLists lists of resources/policies grouped by the status, this structure is meant for internal use of report handling and not an API
1233
type AllLists struct {
13-
passed []string
14-
failed []string
15-
skipped []string
16-
excluded []string
17-
other []string
34+
itemToStatus map[string]apis.ScanningStatus
35+
passed int
36+
failed int
37+
skipped int
38+
other int
1839
}
1940

20-
type Iterator interface {
21-
HasNext() bool
22-
Next() string
23-
Len() int
41+
func (all *AllLists) Failed() int { return all.failed }
42+
func (all *AllLists) Passed() int { return all.passed }
43+
func (all *AllLists) Skipped() int { return all.skipped }
44+
func (all *AllLists) Other() int { return all.other }
45+
func (all *AllLists) Len() int {
46+
return all.failed + all.passed + all.skipped + all.other
2447
}
25-
26-
type AllListsIterator struct {
27-
allLists *AllLists
28-
size int
29-
index int
30-
failedIndex int
31-
passIndex int
32-
skippedIndex int
33-
otherIndex int
48+
func (all *AllLists) All() map[string]apis.ScanningStatus {
49+
return all.itemToStatus
3450
}
3551

36-
func (all *AllLists) createIterator() Iterator {
37-
return &AllListsIterator{
38-
size: len(all.failed) + len(all.passed) + len(all.skipped) + len(all.other),
39-
allLists: all,
52+
// Initialize initialize the AllLists object map with the given size - this is an optimization for the map
53+
func (all *AllLists) Initialize(size int) {
54+
if all.itemToStatus == nil {
55+
all.itemToStatus = make(map[string]apis.ScanningStatus, size)
4056
}
4157
}
4258

43-
func (iter *AllListsIterator) Len() int {
44-
return iter.size
59+
// Clear remove all items and reset the counters
60+
func (all *AllLists) Clear() {
61+
if all.itemToStatus != nil {
62+
maps.Clear(all.itemToStatus)
63+
all.passed = 0
64+
all.failed = 0
65+
all.skipped = 0
66+
all.other = 0
67+
}
4568
}
4669

47-
func (iter *AllListsIterator) HasNext() bool {
48-
return iter.index < iter.size
49-
}
70+
// Append append single string to matching status list
71+
func (all *AllLists) Append(status apis.ScanningStatus, str ...string) {
72+
if all.itemToStatus == nil {
73+
all.itemToStatus = make(map[string]apis.ScanningStatus, len(str))
74+
}
5075

51-
func (iter *AllListsIterator) Next() string {
52-
var item string
53-
if iter.HasNext() {
54-
if iter.failedIndex < len(iter.allLists.failed) {
55-
item = iter.allLists.failed[iter.failedIndex]
56-
iter.failedIndex++
57-
} else if iter.passIndex < len(iter.allLists.passed) {
58-
item = iter.allLists.passed[iter.passIndex]
59-
iter.passIndex++
60-
} else if iter.skippedIndex < len(iter.allLists.skipped) {
61-
item = iter.allLists.skipped[iter.skippedIndex]
62-
iter.skippedIndex++
63-
} else if iter.otherIndex < len(iter.allLists.other) {
64-
item = iter.allLists.other[iter.otherIndex]
65-
iter.otherIndex++
76+
for _, s := range str {
77+
oldStatus, exist := all.itemToStatus[s]
78+
if !exist {
79+
all.itemToStatus[s] = status
80+
switch status {
81+
case apis.StatusPassed:
82+
all.passed++
83+
case apis.StatusFailed:
84+
all.failed++
85+
case apis.StatusSkipped:
86+
all.skipped++
87+
default:
88+
all.other++
89+
}
90+
// element exist with different status
91+
} else if oldStatus != status {
92+
// check if the new status is more significant
93+
if result := apis.Compare(oldStatus, status); result == status {
94+
all.itemToStatus[s] = status
95+
switch status {
96+
case apis.StatusPassed:
97+
all.passed++
98+
case apis.StatusFailed:
99+
all.failed++
100+
case apis.StatusSkipped:
101+
all.skipped++
102+
default:
103+
all.other++
104+
}
105+
106+
// update the old status
107+
switch oldStatus {
108+
case apis.StatusPassed:
109+
all.passed--
110+
case apis.StatusFailed:
111+
all.failed--
112+
case apis.StatusSkipped:
113+
all.skipped--
114+
default:
115+
all.other--
116+
}
117+
}
66118
}
67-
iter.index++
68119
}
69-
return item
70120
}
71121

72-
// GetAllResources
73-
74-
func (all *AllLists) Failed() []string { return all.failed }
75-
func (all *AllLists) Passed() []string { return append(all.passed, all.excluded...) }
76-
func (all *AllLists) Skipped() []string { return all.skipped }
77-
func (all *AllLists) Other() []string { return all.other }
78-
func (all *AllLists) All() Iterator {
79-
return all.createIterator()
122+
// Update AllLists objects with
123+
func (all *AllLists) Update(all2 *AllLists) {
124+
for item, status := range all2.itemToStatus {
125+
all.Append(apis.ScanningStatus(status), item)
126+
}
80127
}
81128

82-
// Append append single string to matching status list
83-
func (all *AllLists) Append(status apis.ScanningStatus, str ...string) {
129+
func (all *AllLists) GetItems(status apis.ScanningStatus) []string {
130+
var amount int
84131
switch status {
85132
case apis.StatusPassed:
86-
all.passed = append(all.passed, str...)
87-
case apis.StatusSkipped:
88-
all.skipped = append(all.skipped, str...)
133+
amount = all.passed
89134
case apis.StatusFailed:
90-
all.failed = append(all.failed, str...)
135+
amount = all.failed
136+
case apis.StatusSkipped:
137+
amount = all.skipped
91138
default:
92-
all.other = append(all.other, str...)
139+
amount = all.other
93140
}
94-
}
95-
96-
// Update AllLists objects with
97-
func (all *AllLists) Update(all2 *AllLists) {
98-
all.passed = append(all.passed, all2.passed...)
99-
all.skipped = append(all.skipped, all2.skipped...)
100-
all.failed = append(all.failed, all2.failed...)
101-
all.other = append(all.other, all2.other...)
102-
}
103141

104-
// ToUnique - Call this function only when setting the List
105-
func (all *AllLists) toUniqueBase() {
106-
// remove duplications from each resource list
107-
all.failed = slices.UniqueStrings(all.failed)
108-
all.passed = slices.UniqueStrings(all.passed)
109-
all.skipped = slices.UniqueStrings(all.skipped)
110-
all.other = slices.UniqueStrings(all.other)
111-
}
112-
113-
// ToUnique - Call this function only when setting the List
114-
func (all *AllLists) ToUniqueControls() {
115-
all.toUniqueBase()
116-
}
117-
118-
// ToUnique - Call this function only when setting the List
119-
func (all *AllLists) ToUniqueResources() {
120-
all.failed = slices.UniqueStrings(all.failed)
121-
122-
const heuristicCapacity = 100 // alloc 100 slots to the stack. The rest would go to the heap - see https://github.com/golang/go/issues/58215
123-
124-
trimmed := append(make([]string, 0, heuristicCapacity), make([]string, 0, max(len(all.failed)+len(all.excluded)+len(all.passed)+len(all.skipped), heuristicCapacity)-heuristicCapacity)...)
125-
126-
// remove failed from excluded list
127-
trimmed = append(trimmed, all.failed...)
128-
all.skipped = slices.TrimStableUnique(all.skipped, trimmed)
129-
130-
// remove failed and skipped from passed list
131-
trimmed = append(trimmed, all.skipped...)
132-
all.passed = slices.TrimStableUnique(all.passed, trimmed)
133-
134-
// remove failed, skipped and passed from "other" list
135-
trimmed = append(trimmed, all.passed...)
136-
all.other = slices.TrimStableUnique(all.other, trimmed)
137-
}
142+
if amount == 0 {
143+
return []string{}
144+
}
138145

139-
func max(a, b int) int {
140-
if a > b {
141-
return a
146+
items := make([]string, 0, amount)
147+
for item, itemStatus := range all.itemToStatus {
148+
if itemStatus == status {
149+
items = append(items, item)
150+
}
142151
}
143152

144-
return b
153+
return items
145154
}
Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1 @@
11
package helpers
2-
3-
import (
4-
"testing"
5-
6-
"github.com/kubescape/opa-utils/reporthandling/apis"
7-
)
8-
9-
func BenchmarkToUniqueResources(b *testing.B) {
10-
listA := mockAllListsA()
11-
listA.Append(apis.StatusExcluded, "b")
12-
listA.Append(apis.StatusExcluded, "b")
13-
listA.Append(apis.StatusExcluded, "b")
14-
b.ResetTimer()
15-
b.ReportAllocs()
16-
17-
for n := 0; n < b.N; n++ {
18-
listA.ToUniqueResources()
19-
}
20-
}
Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
11
package helpers
22

3+
import "github.com/kubescape/opa-utils/reporthandling/apis"
4+
35
func MockAllListsForIntegration() *AllLists {
4-
return &AllLists{
5-
passed: []string{"a", "b"},
6-
failed: []string{"a", "e"},
7-
skipped: []string{"f"},
8-
other: []string{"i", "g", "h", "i"},
9-
}
6+
mock := &AllLists{}
7+
mock.Append(apis.StatusPassed, "a", "b")
8+
mock.Append(apis.StatusFailed, "a", "e")
9+
mock.Append(apis.StatusSkipped, "f")
10+
mock.Append(apis.StatusUnknown, "i", "g", "h", "i")
11+
12+
return mock
1013
}

0 commit comments

Comments
 (0)