Skip to content

Commit 841f485

Browse files
agarciamontoroagarciamontoro
authored
Add config example files for release testing and performance comparisons (#977)
* Add validation command to check-style It makes sure that the example files in examples/config/{release,perfcomp}/json are valid JSON. * Add JSON to examples/config/{release,perfcomp} These files are the templates we've been using for both release testing of the tool and the performance comparisons. They have been trimmed down to only include non-default fields, except for some fields that even if they are equal to the default value, they are kept for reference. * Unify perfcomp and release config files And remove duplicated files
1 parent d5459d0 commit 841f485

File tree

18 files changed

+577
-1
lines changed

18 files changed

+577
-1
lines changed

‎Makefile‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ verify-gomod: ## Run go mod verify.
9393
$(GO) mod download
9494
$(GO) mod verify
9595

96-
check-style: golangci-lint validate-json-configs ## Check the style of the code.
96+
check-style: golangci-lint validate-json-configs validate-example-configs ## Check the style of the code.
9797

9898
golangci-lint:
9999
$(GO) install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.64
@@ -111,6 +111,16 @@ validate-json-configs:
111111
$(GO) run ./scripts/json_validator.go config/simulcontroller.sample.json
112112
$(GO) run ./scripts/json_validator.go config/browsercontroller.sample.json
113113

114+
validate-example-configs: ## Validate example config files against real config structs.
115+
$(GO) run ./cmd/validate-examples --type comparison examples/config/release/json/comparison.json
116+
$(GO) run ./cmd/validate-examples --type config examples/config/release/json/config.json
117+
$(GO) run ./cmd/validate-examples --type coordinator examples/config/release/json/coordinator.json
118+
$(GO) run ./cmd/validate-examples --type deployer examples/config/release/json/deployer.json
119+
$(GO) run ./cmd/validate-examples --type comparison examples/config/perfcomp/json/comparison.json
120+
$(GO) run ./cmd/validate-examples --type config examples/config/perfcomp/json/config.json
121+
$(GO) run ./cmd/validate-examples --type coordinator examples/config/perfcomp/json/coordinator.json
122+
$(GO) run ./cmd/validate-examples --type deployer examples/config/perfcomp/json/deployer.json
123+
114124
test: ## Run all tests.
115125
$(GO) test -v -mod=readonly -race -tags=integration ./...
116126

‎cmd/validate-examples/main.go‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
package main
2+
3+
import (
4+
"flag"
5+
"fmt"
6+
"os"
7+
8+
"github.com/mattermost/mattermost-load-test-ng/comparison"
9+
"github.com/mattermost/mattermost-load-test-ng/coordinator"
10+
"github.com/mattermost/mattermost-load-test-ng/defaults"
11+
"github.com/mattermost/mattermost-load-test-ng/deployment"
12+
"github.com/mattermost/mattermost-load-test-ng/loadtest"
13+
)
14+
15+
func main() {
16+
configType := flag.String("type", "", "config type: config, comparison, coordinator, deployer")
17+
flag.Parse()
18+
19+
if *configType == "" {
20+
fmt.Fprintf(os.Stderr, "error: --type is required\n")
21+
os.Exit(1)
22+
}
23+
24+
args := flag.Args()
25+
if len(args) != 1 {
26+
fmt.Fprintf(os.Stderr, "error: exactly one file path argument is required\n")
27+
os.Exit(1)
28+
}
29+
filePath := args[0]
30+
31+
var cfg any
32+
var err error
33+
34+
switch *configType {
35+
case "config":
36+
cfg, err = loadtest.ReadConfig(filePath)
37+
case "comparison":
38+
cfg, err = comparison.ReadConfig(filePath)
39+
case "coordinator":
40+
cfg, err = coordinator.ReadConfig(filePath)
41+
case "deployer":
42+
cfg, err = deployment.ReadConfig(filePath)
43+
default:
44+
fmt.Fprintf(os.Stderr, "error: unknown config type %q\n", *configType)
45+
os.Exit(1)
46+
}
47+
48+
if err != nil {
49+
fmt.Fprintf(os.Stderr, "error: ReadConfig failed for %s: %v\n", filePath, err)
50+
os.Exit(1)
51+
}
52+
53+
if err := defaults.Validate(cfg); err != nil {
54+
fmt.Fprintf(os.Stderr, "error: validation failed for %s: %v\n", filePath, err)
55+
os.Exit(1)
56+
}
57+
58+
fmt.Printf("ok: %s (%s)\n", filePath, *configType)
59+
}

‎examples/config/perfcomp/json/comparison.json‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
{
2+
"BaseBuild": {
3+
"Label": "release-TBD",
4+
"URL": "https://releases.mattermost.com/TBD/mattermost-enterprise-TBD-linux-amd64.tar.gz"
5+
},
6+
"NewBuild": {
7+
"Label": "release-TBD-rc1",
8+
"URL": "https://releases.mattermost.com/TBD-rc1/mattermost-enterprise-TBD-rc1-linux-amd64.tar.gz"
9+
},
10+
"LoadTests": [
11+
{
12+
"DBDumpURL": "https://lt-public-data.s3.amazonaws.com/12M_610_fixed_psql.sql.gz",
13+
"DBEngine": "postgresql",
14+
"Type": "unbounded"
15+
},
16+
{
17+
"DBDumpURL": "https://lt-public-data.s3.amazonaws.com/12M_610_fixed_psql.sql.gz",
18+
"DBEngine": "postgresql",
19+
"Duration": "90m",
20+
"NumUsers": 6500,
21+
"Type": "bounded"
22+
}
23+
],
24+
"Output": {
25+
"GenerateGraphs": true
26+
}
27+
}

‎examples/config/perfcomp/json/config.json‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
{
2+
"BrowserLogSettings": {
3+
"ConsoleLevel": "debug",
4+
"EnableConsole": true
5+
},
6+
"InstanceConfiguration": {
7+
"NumChannels": 0,
8+
"PercentDirectChannels": 0,
9+
"PercentGroupChannels": 0,
10+
"PercentPrivateChannels": 0,
11+
"PercentPublicChannels": 1
12+
},
13+
"LogSettings": {
14+
"ConsoleLevel": "DEBUG",
15+
"EnableColor": true,
16+
"FileJson": true,
17+
"FileLevel": "DEBUG",
18+
"FileLocation": "ltagent.log"
19+
},
20+
"UserControllerConfiguration": {
21+
"Type": "simulative",
22+
"RatesDistribution": [
23+
{
24+
"Percentage": 0.05,
25+
"Rate": 1.0
26+
},
27+
{
28+
"Percentage": 0.1,
29+
"Rate": 2.0
30+
},
31+
{
32+
"Percentage": 0.15,
33+
"Rate": 3.0
34+
},
35+
{
36+
"Percentage": 0.4,
37+
"Rate": 6.0
38+
},
39+
{
40+
"Percentage": 0.3,
41+
"Rate": 30.0
42+
}
43+
]
44+
},
45+
"UsersConfiguration": {
46+
"MaxActiveBrowserUsers": 0
47+
}
48+
}

‎examples/config/perfcomp/json/coordinator.json‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
{
2+
"NumUsersDec": 6,
3+
"NumUsersInc": 6,
4+
"ClusterConfig": {
5+
"MaxActiveUsers": 20000,
6+
"BrowserAgents": []
7+
},
8+
"LogSettings": {
9+
"ConsoleLevel": "INFO",
10+
"FileJson": true,
11+
"FileLevel": "INFO",
12+
"FileLocation": "ltcoordinator.log"
13+
},
14+
"MonitorConfig": {
15+
"Queries": [
16+
{
17+
"Alert": true,
18+
"Description": "Percentage of HTTP 5xx server errors",
19+
"Legend": "Percent",
20+
"MinIntervalSec": 60,
21+
"Query": "(sum(rate(mattermost_api_time_count{status_code=~\"5..\"}[1m]))/sum(rate(mattermost_api_time_count[1m])))*100",
22+
"Threshold": 0.025
23+
},
24+
{
25+
"Alert": true,
26+
"Description": "Average client request duration",
27+
"Legend": "Avg duration (s)",
28+
"MinIntervalSec": 60,
29+
"Query": "sum(rate(loadtest_http_request_time_sum[1m]))/sum(rate(loadtest_http_request_time_count[1m]))",
30+
"Threshold": 0.1
31+
},
32+
{
33+
"Alert": true,
34+
"Description": "99th percentile of client request duration",
35+
"Legend": "P99 duration (s)",
36+
"MinIntervalSec": 60,
37+
"Query": "histogram_quantile(0.99, sum(rate(loadtest_http_request_time_bucket[1m])) by (le))",
38+
"Threshold": 2
39+
},
40+
{
41+
"Alert": true,
42+
"Description": "Percentage of HTTP 5xx client errors",
43+
"Legend": "Percent",
44+
"MinIntervalSec": 60,
45+
"Query": "(sum(rate(loadtest_http_errors_total{status_code=~\"5..\"}[1m]))/sum(rate(loadtest_http_request_time_count[1m])))*100",
46+
"Threshold": 0.025
47+
},
48+
{
49+
"Alert": true,
50+
"Description": "Percentage of client timeouts",
51+
"Legend": "Percent",
52+
"MinIntervalSec": 60,
53+
"Query": "(sum(rate(loadtest_http_timeouts_total[1m]))/sum(rate(loadtest_http_request_time_count[1m]))) * 100",
54+
"Threshold": 0.025
55+
},
56+
{
57+
"Alert": true,
58+
"Description": "CPU utilization - Average of app nodes",
59+
"Legend": "Percent",
60+
"MinIntervalSec": 60,
61+
"Query": "100 - 100 * (avg(irate(node_cpu_seconds_total{instance=~\"app.*\",mode=\"idle\"}[5m])))",
62+
"Threshold": 85
63+
},
64+
{
65+
"Alert": true,
66+
"Description": "Memory utilization - Average of app nodes",
67+
"Legend": "Percent",
68+
"MinIntervalSec": 60,
69+
"Query": "100 - 100 * avg(node_memory_MemAvailable_bytes{instance=~\"app.*\"} / node_memory_MemTotal_bytes{instance=~\"app.*\"})",
70+
"Threshold": 85
71+
},
72+
{
73+
"Alert": true,
74+
"Description": "Percentage of TCP retransmissions in the app nodes",
75+
"Legend": "Percent",
76+
"MinIntervalSec": 60,
77+
"Query": "(avg(rate(node_netstat_Tcp_RetransSegs{instance=~\"app.*\"}[1m])) / avg(rate(node_netstat_Tcp_OutSegs{instance=~\"app.*\"}[1m]))) * 100",
78+
"Threshold": 0.5
79+
},
80+
{
81+
"Alert": true,
82+
"Description": "Percentage of TCP retransmissions in the proxy node",
83+
"Legend": "Percent",
84+
"MinIntervalSec": 60,
85+
"Query": "(avg(rate(node_netstat_Tcp_RetransSegs{instance=~\"proxy:9100\"}[1m])) / avg(rate(node_netstat_Tcp_OutSegs{instance=~\"proxy:9100\"}[1m]))) * 100",
86+
"Threshold": 0.5
87+
}
88+
]
89+
}
90+
}

‎examples/config/perfcomp/json/deployer.json‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
{
2+
"AWSAMI": "ami-003d3d03cfe1b0468",
3+
"AWSAvailabilityZone": "",
4+
"AWSProfile": "mm-loadtest",
5+
"AgentInstanceCount": 11,
6+
"AgentInstanceType": "c7i.xlarge",
7+
"BrowserAgentInstanceCount": 0,
8+
"BrowserAgentInstanceType": "c7i.xlarge",
9+
"AppInstanceCount": 2,
10+
"AppInstanceType": "c7i.2xlarge",
11+
"ClusterName": "TBD",
12+
"LoadTestDownloadURL": "https://github.com/mattermost/mattermost-load-test-ng/releases/download/vTBD/mattermost-load-test-ng-vTBD-linux-amd64.tar.gz",
13+
"MattermostDownloadURL": "https://latest.mattermost.com/mattermost-enterprise-linux",
14+
"MattermostLicenseFile": "",
15+
"MetricsInstanceType": "t3.xlarge",
16+
"ProxyInstanceCount": 1,
17+
"ProxyInstanceType": "c7i.xlarge",
18+
"SSHPublicKey": "~/.ssh/id_ed25519.pub",
19+
"TerraformStateDir": "./ltstatus",
20+
"LogSettings": {
21+
"ConsoleLevel": "DEBUG",
22+
"EnableColor": true,
23+
"FileJson": true,
24+
"FileLevel": "DEBUG",
25+
"FileLocation": "deployer.log"
26+
},
27+
"Report": {
28+
"GraphQueries": [
29+
{
30+
"Name": "CPU Utilization",
31+
"Query": "avg(rate(mattermost_process_cpu_seconds_total{instance=~\"app.*\"}[1m])* 100)"
32+
},
33+
{
34+
"Name": "Heap In Use",
35+
"Query": "avg(go_memstats_heap_inuse_bytes{instance=~\"app.*:8067\"})"
36+
},
37+
{
38+
"Name": "Stack In Use",
39+
"Query": "avg(go_memstats_stack_inuse_bytes{instance=~\"app.*:8067\"})"
40+
},
41+
{
42+
"Name": "Goroutines In Use",
43+
"Query": "sum(go_goroutines{instance=~\"app.*:8067\"})"
44+
},
45+
{
46+
"Name": "RPS",
47+
"Query": "sum(rate(mattermost_http_requests_total{instance=~\"app.*:8067\"}[1m]))"
48+
},
49+
{
50+
"Name": "Avg Store times",
51+
"Query": "sum(rate(mattermost_db_store_time_sum{instance=~\"app.*:8067\"}[1m])) / sum(rate(mattermost_db_store_time_count{instance=~\"app.*:8067\"}[1m]))"
52+
},
53+
{
54+
"Name": "P99 Store times",
55+
"Query": "histogram_quantile(0.99, sum(rate(mattermost_db_store_time_bucket[1m])) by (le))"
56+
},
57+
{
58+
"Name": "Avg API times",
59+
"Query": "sum(rate(mattermost_api_time_sum[1m])) / sum(rate(mattermost_api_time_count[1m]))"
60+
},
61+
{
62+
"Name": "P99 API times",
63+
"Query": "histogram_quantile(0.99, sum(rate(mattermost_api_time_bucket[1m])) by (le))"
64+
},
65+
{
66+
"Name": "Number of Connected Devices",
67+
"Query": "sum(mattermost_http_websockets_total{instance=~\"app.*:8067\"})"
68+
}
69+
],
70+
"Label": "{instance=~\"app.*\"}"
71+
},
72+
"SiteURL": "",
73+
"StorageSizes": {
74+
"App": 20
75+
},
76+
"TerraformDBSettings": {
77+
"DBParameters": [],
78+
"InstanceEngine": "aurora-postgresql",
79+
"InstanceCount": 2,
80+
"InstanceType": "db.r7g.2xlarge"
81+
},
82+
"CustomTags": {
83+
"Origin": "performance-comparison"
84+
}
85+
}

examples/config/perfcomp/comparison.toml renamed to examples/config/perfcomp/toml/comparison.toml

File renamed without changes.

examples/config/perfcomp/config.toml renamed to examples/config/perfcomp/toml/config.toml

File renamed without changes.

examples/config/perfcomp/coordinator.toml renamed to examples/config/perfcomp/toml/coordinator.toml

File renamed without changes.

examples/config/perfcomp/deployer.toml renamed to examples/config/perfcomp/toml/deployer.toml

File renamed without changes.

0 commit comments

Comments
 (0)