Skip to content

Commit d817584

Browse files
mlsmayconmlsmaycon
authored
[misc] fix Windows client and management bench tests (#4424)
Windows tests had too many directories, causing issues to the payload via psexec. Also migrated all checked benchmarks to send data to grafana.
1 parent 4d3dc34 commit d817584

File tree

12 files changed

+282
-222
lines changed

12 files changed

+282
-222
lines changed

‎.github/workflows/golang-test-linux.yml‎

Lines changed: 31 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -382,6 +382,32 @@ jobs:
382382
store: [ 'sqlite', 'postgres' ]
383383
runs-on: ubuntu-22.04
384384
steps:
385+
- name: Create Docker network
386+
run: docker network create promnet
387+
388+
- name: Start Prometheus Pushgateway
389+
run: docker run -d --name pushgateway --network promnet -p 9091:9091 prom/pushgateway
390+
391+
- name: Start Prometheus (for Pushgateway forwarding)
392+
run: |
393+
echo '
394+
global:
395+
scrape_interval: 15s
396+
scrape_configs:
397+
- job_name: "pushgateway"
398+
static_configs:
399+
- targets: ["pushgateway:9091"]
400+
remote_write:
401+
- url: ${{ secrets.GRAFANA_URL }}
402+
basic_auth:
403+
username: ${{ secrets.GRAFANA_USER }}
404+
password: ${{ secrets.GRAFANA_API_KEY }}
405+
' > prometheus.yml
406+
407+
docker run -d --name prometheus --network promnet \
408+
-v $PWD/prometheus.yml:/etc/prometheus/prometheus.yml \
409+
-p 9090:9090 \
410+
prom/prometheus
385411
- name: Install Go
386412
uses: actions/setup-go@v5
387413
with:
@@ -428,9 +454,10 @@ jobs:
428454
CGO_ENABLED=1 GOARCH=${{ matrix.arch }} \
429455
NETBIRD_STORE_ENGINE=${{ matrix.store }} \
430456
CI=true \
457+
GIT_BRANCH=${{ github.ref_name }} \
431458
go test -tags devcert -run=^$ -bench=. \
432-
-exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' \
433-
-timeout 20m ./management/... ./shared/management/...
459+
-exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE,GIT_BRANCH,GITHUB_RUN_ID' \
460+
-timeout 20m ./management/... ./shared/management/... $(go list ./management/... ./shared/management/... | grep -v -e /management/server/http)
434461
435462
api_benchmark:
436463
name: "Management / Benchmark (API)"
@@ -521,7 +548,7 @@ jobs:
521548
-run=^$ \
522549
-bench=. \
523550
-exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE,GIT_BRANCH,GITHUB_RUN_ID' \
524-
-timeout 20m ./management/... ./shared/management/...
551+
-timeout 20m ./management/server/http/...
525552
526553
api_integration_test:
527554
name: "Management / Integration"
@@ -571,4 +598,4 @@ jobs:
571598
CI=true \
572599
go test -tags=integration \
573600
-exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' \
574-
-timeout 20m ./management/... ./shared/management/...
601+
-timeout 20m ./management/server/http/...

‎.github/workflows/golang-test-windows.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ jobs:
6363
- run: PsExec64 -s -w ${{ github.workspace }} C:\hostedtoolcache\windows\go\${{ steps.go.outputs.go-version }}\x64\bin\go.exe env -w GOMODCACHE=${{ env.cache }}
6464
- run: PsExec64 -s -w ${{ github.workspace }} C:\hostedtoolcache\windows\go\${{ steps.go.outputs.go-version }}\x64\bin\go.exe env -w GOCACHE=${{ env.modcache }}
6565
- run: PsExec64 -s -w ${{ github.workspace }} C:\hostedtoolcache\windows\go\${{ steps.go.outputs.go-version }}\x64\bin\go.exe mod tidy
66-
- run: echo "files=$(go list ./... | ForEach-Object { $_ } | Where-Object { $_ -notmatch '/management' })" >> $env:GITHUB_ENV
66+
- run: echo "files=$(go list ./... | ForEach-Object { $_ } | Where-Object { $_ -notmatch '/management' } | Where-Object { $_ -notmatch '/relay' } | Where-Object { $_ -notmatch '/signal' })" >> $env:GITHUB_ENV
6767

6868
- name: test
6969
run: PsExec64 -s -w ${{ github.workspace }} cmd.exe /c "C:\hostedtoolcache\windows\go\${{ steps.go.outputs.go-version }}\x64\bin\go.exe test -tags=devcert -timeout 10m -p 1 ${{ env.files }} > test-out.txt 2>&1"

‎management/README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,3 +111,6 @@ Generate gRpc code:
111111
#!/bin/bash
112112
protoc -I proto/ proto/management.proto --go_out=. --go-grpc_out=.
113113
```
114+
115+
116+

‎management/server/account_test.go‎

Lines changed: 36 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"time"
1616

1717
"github.com/golang/mock/gomock"
18+
"github.com/prometheus/client_golang/prometheus/push"
1819
log "github.com/sirupsen/logrus"
1920
"github.com/stretchr/testify/assert"
2021
"github.com/stretchr/testify/require"
@@ -25,6 +26,7 @@ import (
2526
"github.com/netbirdio/netbird/management/server/activity"
2627
"github.com/netbirdio/netbird/management/server/cache"
2728
nbcontext "github.com/netbirdio/netbird/management/server/context"
29+
"github.com/netbirdio/netbird/management/server/http/testing/testing_tools"
2830
"github.com/netbirdio/netbird/management/server/idp"
2931
"github.com/netbirdio/netbird/management/server/integrations/port_forwarding"
3032
resourceTypes "github.com/netbirdio/netbird/management/server/networks/resources/types"
@@ -3046,19 +3048,14 @@ func BenchmarkSyncAndMarkPeer(b *testing.B) {
30463048
msPerOp := float64(duration.Nanoseconds()) / float64(b.N) / 1e6
30473049
b.ReportMetric(msPerOp, "ms/op")
30483050

3049-
minExpected := bc.minMsPerOpLocal
30503051
maxExpected := bc.maxMsPerOpLocal
30513052
if os.Getenv("CI") == "true" {
3052-
minExpected = bc.minMsPerOpCICD
30533053
maxExpected = bc.maxMsPerOpCICD
3054+
testing_tools.EvaluateBenchmarkResults(b, bc.name, time.Since(start), "sync", "syncAndMark")
30543055
}
30553056

3056-
if msPerOp < minExpected {
3057-
b.Fatalf("Benchmark %s failed: too fast (%.2f ms/op, minimum %.2f ms/op)", bc.name, msPerOp, minExpected)
3058-
}
3059-
3060-
if msPerOp > (maxExpected * 1.1) {
3061-
b.Fatalf("Benchmark %s failed: too slow (%.2f ms/op, maximum %.2f ms/op)", bc.name, msPerOp, maxExpected)
3057+
if msPerOp > maxExpected {
3058+
b.Logf("Benchmark %s: too slow (%.2f ms/op, max %.2f ms/op)", bc.name, msPerOp, maxExpected)
30623059
}
30633060
})
30643061
}
@@ -3121,19 +3118,14 @@ func BenchmarkLoginPeer_ExistingPeer(b *testing.B) {
31213118
msPerOp := float64(duration.Nanoseconds()) / float64(b.N) / 1e6
31223119
b.ReportMetric(msPerOp, "ms/op")
31233120

3124-
minExpected := bc.minMsPerOpLocal
31253121
maxExpected := bc.maxMsPerOpLocal
31263122
if os.Getenv("CI") == "true" {
3127-
minExpected = bc.minMsPerOpCICD
31283123
maxExpected = bc.maxMsPerOpCICD
3124+
testing_tools.EvaluateBenchmarkResults(b, bc.name, time.Since(start), "login", "existingPeer")
31293125
}
31303126

3131-
if msPerOp < minExpected {
3132-
b.Fatalf("Benchmark %s failed: too fast (%.2f ms/op, minimum %.2f ms/op)", bc.name, msPerOp, minExpected)
3133-
}
3134-
3135-
if msPerOp > (maxExpected * 1.1) {
3136-
b.Fatalf("Benchmark %s failed: too slow (%.2f ms/op, maximum %.2f ms/op)", bc.name, msPerOp, maxExpected)
3127+
if msPerOp > maxExpected {
3128+
b.Logf("Benchmark %s: too slow (%.2f ms/op, max %.2f ms/op)", bc.name, msPerOp, maxExpected)
31373129
}
31383130
})
31393131
}
@@ -3196,24 +3188,44 @@ func BenchmarkLoginPeer_NewPeer(b *testing.B) {
31963188
msPerOp := float64(duration.Nanoseconds()) / float64(b.N) / 1e6
31973189
b.ReportMetric(msPerOp, "ms/op")
31983190

3199-
minExpected := bc.minMsPerOpLocal
32003191
maxExpected := bc.maxMsPerOpLocal
32013192
if os.Getenv("CI") == "true" {
3202-
minExpected = bc.minMsPerOpCICD
32033193
maxExpected = bc.maxMsPerOpCICD
3194+
testing_tools.EvaluateBenchmarkResults(b, bc.name, time.Since(start), "login", "newPeer")
32043195
}
32053196

3206-
if msPerOp < minExpected {
3207-
b.Fatalf("Benchmark %s failed: too fast (%.2f ms/op, minimum %.2f ms/op)", bc.name, msPerOp, minExpected)
3208-
}
3209-
3210-
if msPerOp > (maxExpected * 1.1) {
3211-
b.Fatalf("Benchmark %s failed: too slow (%.2f ms/op, maximum %.2f ms/op)", bc.name, msPerOp, maxExpected)
3197+
if msPerOp > maxExpected {
3198+
b.Logf("Benchmark %s: too slow (%.2f ms/op, max %.2f ms/op)", bc.name, msPerOp, maxExpected)
32123199
}
32133200
})
32143201
}
32153202
}
32163203

3204+
func TestMain(m *testing.M) {
3205+
exitCode := m.Run()
3206+
3207+
if exitCode == 0 && os.Getenv("CI") == "true" {
3208+
runID := os.Getenv("GITHUB_RUN_ID")
3209+
storeEngine := os.Getenv("NETBIRD_STORE_ENGINE")
3210+
err := push.New("http://localhost:9091", "account_manager_benchmark").
3211+
Collector(testing_tools.BenchmarkDuration).
3212+
Grouping("ci_run", runID).
3213+
Grouping("store_engine", storeEngine).
3214+
Push()
3215+
if err != nil {
3216+
log.Printf("Failed to push metrics: %v", err)
3217+
} else {
3218+
time.Sleep(1 * time.Minute)
3219+
_ = push.New("http://localhost:9091", "account_manager_benchmark").
3220+
Grouping("ci_run", runID).
3221+
Grouping("store_engine", storeEngine).
3222+
Delete()
3223+
}
3224+
}
3225+
3226+
os.Exit(exitCode)
3227+
}
3228+
32173229
func Test_GetCreateAccountByPrivateDomain(t *testing.T) {
32183230
manager, err := createManager(t)
32193231
if err != nil {

‎management/server/http/testing/benchmarks/peers_handler_benchmark_test.go‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,9 @@ import (
1717
"github.com/stretchr/testify/assert"
1818

1919
"github.com/netbirdio/netbird/management/server"
20-
"github.com/netbirdio/netbird/shared/management/http/api"
2120
"github.com/netbirdio/netbird/management/server/http/testing/testing_tools"
21+
"github.com/netbirdio/netbird/management/server/http/testing/testing_tools/channel"
22+
"github.com/netbirdio/netbird/shared/management/http/api"
2223
)
2324

2425
const modulePeers = "peers"
@@ -47,7 +48,7 @@ func BenchmarkUpdatePeer(b *testing.B) {
4748

4849
for name, bc := range benchCasesPeers {
4950
b.Run(name, func(b *testing.B) {
50-
apiHandler, am, _ := testing_tools.BuildApiBlackBoxWithDBState(b, "../testdata/peers.sql", nil, false)
51+
apiHandler, am, _ := channel.BuildApiBlackBoxWithDBState(b, "../testdata/peers.sql", nil, false)
5152
testing_tools.PopulateTestData(b, am.(*server.DefaultAccountManager), bc.Peers, bc.Groups, bc.Users, bc.SetupKeys)
5253

5354
b.ResetTimer()
@@ -65,7 +66,7 @@ func BenchmarkUpdatePeer(b *testing.B) {
6566
apiHandler.ServeHTTP(recorder, req)
6667
}
6768

68-
testing_tools.EvaluateBenchmarkResults(b, name, time.Since(start), recorder, modulePeers, testing_tools.OperationUpdate)
69+
testing_tools.EvaluateAPIBenchmarkResults(b, name, time.Since(start), recorder, modulePeers, testing_tools.OperationUpdate)
6970
})
7071
}
7172
}
@@ -82,7 +83,7 @@ func BenchmarkGetOnePeer(b *testing.B) {
8283

8384
for name, bc := range benchCasesPeers {
8485
b.Run(name, func(b *testing.B) {
85-
apiHandler, am, _ := testing_tools.BuildApiBlackBoxWithDBState(b, "../testdata/peers.sql", nil, false)
86+
apiHandler, am, _ := channel.BuildApiBlackBoxWithDBState(b, "../testdata/peers.sql", nil, false)
8687
testing_tools.PopulateTestData(b, am.(*server.DefaultAccountManager), bc.Peers, bc.Groups, bc.Users, bc.SetupKeys)
8788

8889
b.ResetTimer()
@@ -92,7 +93,7 @@ func BenchmarkGetOnePeer(b *testing.B) {
9293
apiHandler.ServeHTTP(recorder, req)
9394
}
9495

95-
testing_tools.EvaluateBenchmarkResults(b, name, time.Since(start), recorder, modulePeers, testing_tools.OperationGetOne)
96+
testing_tools.EvaluateAPIBenchmarkResults(b, name, time.Since(start), recorder, modulePeers, testing_tools.OperationGetOne)
9697
})
9798
}
9899
}
@@ -109,7 +110,7 @@ func BenchmarkGetAllPeers(b *testing.B) {
109110

110111
for name, bc := range benchCasesPeers {
111112
b.Run(name, func(b *testing.B) {
112-
apiHandler, am, _ := testing_tools.BuildApiBlackBoxWithDBState(b, "../testdata/peers.sql", nil, false)
113+
apiHandler, am, _ := channel.BuildApiBlackBoxWithDBState(b, "../testdata/peers.sql", nil, false)
113114
testing_tools.PopulateTestData(b, am.(*server.DefaultAccountManager), bc.Peers, bc.Groups, bc.Users, bc.SetupKeys)
114115

115116
b.ResetTimer()
@@ -119,7 +120,7 @@ func BenchmarkGetAllPeers(b *testing.B) {
119120
apiHandler.ServeHTTP(recorder, req)
120121
}
121122

122-
testing_tools.EvaluateBenchmarkResults(b, name, time.Since(start), recorder, modulePeers, testing_tools.OperationGetAll)
123+
testing_tools.EvaluateAPIBenchmarkResults(b, name, time.Since(start), recorder, modulePeers, testing_tools.OperationGetAll)
123124
})
124125
}
125126
}
@@ -136,7 +137,7 @@ func BenchmarkDeletePeer(b *testing.B) {
136137

137138
for name, bc := range benchCasesPeers {
138139
b.Run(name, func(b *testing.B) {
139-
apiHandler, am, _ := testing_tools.BuildApiBlackBoxWithDBState(b, "../testdata/peers.sql", nil, false)
140+
apiHandler, am, _ := channel.BuildApiBlackBoxWithDBState(b, "../testdata/peers.sql", nil, false)
140141
testing_tools.PopulateTestData(b, am.(*server.DefaultAccountManager), 1000, bc.Groups, bc.Users, bc.SetupKeys)
141142

142143
b.ResetTimer()
@@ -146,7 +147,7 @@ func BenchmarkDeletePeer(b *testing.B) {
146147
apiHandler.ServeHTTP(recorder, req)
147148
}
148149

149-
testing_tools.EvaluateBenchmarkResults(b, name, time.Since(start), recorder, modulePeers, testing_tools.OperationDelete)
150+
testing_tools.EvaluateAPIBenchmarkResults(b, name, time.Since(start), recorder, modulePeers, testing_tools.OperationDelete)
150151
})
151152
}
152153
}

‎management/server/http/testing/benchmarks/setupkeys_handler_benchmark_test.go‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,9 @@ import (
1717
"github.com/stretchr/testify/assert"
1818

1919
"github.com/netbirdio/netbird/management/server"
20-
"github.com/netbirdio/netbird/shared/management/http/api"
2120
"github.com/netbirdio/netbird/management/server/http/testing/testing_tools"
21+
"github.com/netbirdio/netbird/management/server/http/testing/testing_tools/channel"
22+
"github.com/netbirdio/netbird/shared/management/http/api"
2223
)
2324

2425
// Map to store peers, groups, users, and setupKeys by name
@@ -47,7 +48,7 @@ func BenchmarkCreateSetupKey(b *testing.B) {
4748

4849
for name, bc := range benchCasesSetupKeys {
4950
b.Run(name, func(b *testing.B) {
50-
apiHandler, am, _ := testing_tools.BuildApiBlackBoxWithDBState(b, "../testdata/setup_keys.sql", nil, false)
51+
apiHandler, am, _ := channel.BuildApiBlackBoxWithDBState(b, "../testdata/setup_keys.sql", nil, false)
5152
testing_tools.PopulateTestData(b, am.(*server.DefaultAccountManager), bc.Peers, bc.Groups, bc.Users, bc.SetupKeys)
5253

5354
b.ResetTimer()
@@ -69,7 +70,7 @@ func BenchmarkCreateSetupKey(b *testing.B) {
6970
apiHandler.ServeHTTP(recorder, req)
7071
}
7172

72-
testing_tools.EvaluateBenchmarkResults(b, name, time.Since(start), recorder, moduleSetupKeys, testing_tools.OperationCreate)
73+
testing_tools.EvaluateAPIBenchmarkResults(b, name, time.Since(start), recorder, moduleSetupKeys, testing_tools.OperationCreate)
7374
})
7475
}
7576
}
@@ -86,7 +87,7 @@ func BenchmarkUpdateSetupKey(b *testing.B) {
8687

8788
for name, bc := range benchCasesSetupKeys {
8889
b.Run(name, func(b *testing.B) {
89-
apiHandler, am, _ := testing_tools.BuildApiBlackBoxWithDBState(b, "../testdata/setup_keys.sql", nil, false)
90+
apiHandler, am, _ := channel.BuildApiBlackBoxWithDBState(b, "../testdata/setup_keys.sql", nil, false)
9091
testing_tools.PopulateTestData(b, am.(*server.DefaultAccountManager), bc.Peers, bc.Groups, bc.Users, bc.SetupKeys)
9192

9293
b.ResetTimer()
@@ -109,7 +110,7 @@ func BenchmarkUpdateSetupKey(b *testing.B) {
109110
apiHandler.ServeHTTP(recorder, req)
110111
}
111112

112-
testing_tools.EvaluateBenchmarkResults(b, name, time.Since(start), recorder, moduleSetupKeys, testing_tools.OperationUpdate)
113+
testing_tools.EvaluateAPIBenchmarkResults(b, name, time.Since(start), recorder, moduleSetupKeys, testing_tools.OperationUpdate)
113114
})
114115
}
115116
}
@@ -126,7 +127,7 @@ func BenchmarkGetOneSetupKey(b *testing.B) {
126127

127128
for name, bc := range benchCasesSetupKeys {
128129
b.Run(name, func(b *testing.B) {
129-
apiHandler, am, _ := testing_tools.BuildApiBlackBoxWithDBState(b, "../testdata/setup_keys.sql", nil, false)
130+
apiHandler, am, _ := channel.BuildApiBlackBoxWithDBState(b, "../testdata/setup_keys.sql", nil, false)
130131
testing_tools.PopulateTestData(b, am.(*server.DefaultAccountManager), bc.Peers, bc.Groups, bc.Users, bc.SetupKeys)
131132

132133
b.ResetTimer()
@@ -136,7 +137,7 @@ func BenchmarkGetOneSetupKey(b *testing.B) {
136137
apiHandler.ServeHTTP(recorder, req)
137138
}
138139

139-
testing_tools.EvaluateBenchmarkResults(b, name, time.Since(start), recorder, moduleSetupKeys, testing_tools.OperationGetOne)
140+
testing_tools.EvaluateAPIBenchmarkResults(b, name, time.Since(start), recorder, moduleSetupKeys, testing_tools.OperationGetOne)
140141
})
141142
}
142143
}
@@ -153,7 +154,7 @@ func BenchmarkGetAllSetupKeys(b *testing.B) {
153154

154155
for name, bc := range benchCasesSetupKeys {
155156
b.Run(name, func(b *testing.B) {
156-
apiHandler, am, _ := testing_tools.BuildApiBlackBoxWithDBState(b, "../testdata/setup_keys.sql", nil, false)
157+
apiHandler, am, _ := channel.BuildApiBlackBoxWithDBState(b, "../testdata/setup_keys.sql", nil, false)
157158
testing_tools.PopulateTestData(b, am.(*server.DefaultAccountManager), bc.Peers, bc.Groups, bc.Users, bc.SetupKeys)
158159

159160
b.ResetTimer()
@@ -163,7 +164,7 @@ func BenchmarkGetAllSetupKeys(b *testing.B) {
163164
apiHandler.ServeHTTP(recorder, req)
164165
}
165166

166-
testing_tools.EvaluateBenchmarkResults(b, name, time.Since(start), recorder, moduleSetupKeys, testing_tools.OperationGetAll)
167+
testing_tools.EvaluateAPIBenchmarkResults(b, name, time.Since(start), recorder, moduleSetupKeys, testing_tools.OperationGetAll)
167168
})
168169
}
169170
}
@@ -180,7 +181,7 @@ func BenchmarkDeleteSetupKey(b *testing.B) {
180181

181182
for name, bc := range benchCasesSetupKeys {
182183
b.Run(name, func(b *testing.B) {
183-
apiHandler, am, _ := testing_tools.BuildApiBlackBoxWithDBState(b, "../testdata/setup_keys.sql", nil, false)
184+
apiHandler, am, _ := channel.BuildApiBlackBoxWithDBState(b, "../testdata/setup_keys.sql", nil, false)
184185
testing_tools.PopulateTestData(b, am.(*server.DefaultAccountManager), bc.Peers, bc.Groups, bc.Users, 1000)
185186

186187
b.ResetTimer()
@@ -190,7 +191,7 @@ func BenchmarkDeleteSetupKey(b *testing.B) {
190191
apiHandler.ServeHTTP(recorder, req)
191192
}
192193

193-
testing_tools.EvaluateBenchmarkResults(b, name, time.Since(start), recorder, moduleSetupKeys, testing_tools.OperationDelete)
194+
testing_tools.EvaluateAPIBenchmarkResults(b, name, time.Since(start), recorder, moduleSetupKeys, testing_tools.OperationDelete)
194195
})
195196
}
196197
}

0 commit comments

Comments
 (0)