Skip to content

Commit f3d69ff

Browse files
gheorghestrimtugheorghestrimtu
authored
telemetry down alert tests (#106)
* 2 tests * capitalize OCR * stop chaos
1 parent c3ba755 commit f3d69ff

File tree

8 files changed

+181
-78
lines changed

8 files changed

+181
-78
lines changed

‎environment/environment.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,15 @@ func GetExplorerClientFromEnv(env Environment) (*client.ExplorerClient, error) {
5757
return NewExplorerClient(sd.LocalURL.String())
5858
}
5959

60+
// GetPrometheusClientFromEnv returns a Prometheus client
61+
func GetPrometheusClientFromEnv(env Environment) (*client.Prometheus, error) {
62+
sd, err := env.GetServiceDetails(PrometheusAPIPort)
63+
if err != nil {
64+
return nil, err
65+
}
66+
return client.NewPrometheusClient(sd.LocalURL.String())
67+
}
68+
6069
// GetChainlinkClients will return all instantiated Chainlink clients for a given environment
6170
func GetChainlinkClients(env Environment) ([]client.Chainlink, error) {
6271
var clients []client.Chainlink

‎environment/environment_templates.go‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,13 @@ import (
2020

2121
// Ports for common services
2222
const (
23-
AdapterAPIPort = 6060
24-
ChainlinkWebPort = 6688
25-
ChainlinkP2PPort = 6690
26-
EVMRPCPort = 8545
27-
MinersRPCPort = 9545
28-
ExplorerAPIPort = 8080
23+
AdapterAPIPort = 6060
24+
ChainlinkWebPort = 6688
25+
ChainlinkP2PPort = 6690
26+
EVMRPCPort = 8545
27+
MinersRPCPort = 9545
28+
ExplorerAPIPort = 8080
29+
PrometheusAPIPort = 9090
2930
)
3031

3132
// NewAdapterManifest is the k8s manifest that when used will deploy an external adapter to an environment
@@ -536,7 +537,7 @@ func PrometheusGroup() K8sEnvSpecInit {
536537
return func(config *config.NetworkConfig) (string, K8sEnvSpecs) {
537538
var specs K8sEnvSpecs
538539
prometheusDependencyGroup := &K8sManifestGroup{
539-
id: "PrometheusDependencyGroup",
540+
id: "PrometheusDependencyGroup",
540541
manifests: []K8sEnvResource{NewPrometheusManifest()},
541542
}
542543
specs = append(specs, prometheusDependencyGroup)

‎environment/templates/prometheus/rules/ocr.rules.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
expr: (rate (sum:ocr_telemetry_ingested_total[1m])) == bool 0
2020
- alert: Telemetry Down (infra)
2121
expr: bool:telemetry_down == 1
22-
for: 5m
22+
for: 1m
2323
labels:
2424
severity: critical
2525
team: infra
@@ -28,7 +28,7 @@
2828

2929
- alert: Telemetry Down (o11y)
3030
expr: bool:telemetry_down == 1
31-
for: 5m
31+
for: 1m
3232
labels:
3333
severity: critical
3434
team: monitoring

‎suite/alerts/alerts_test.go‎

Lines changed: 91 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,107 @@
11
package alerts
22

33
import (
4+
"context"
5+
"fmt"
46
. "github.com/onsi/ginkgo"
57
. "github.com/onsi/gomega"
8+
. "github.com/onsi/gomega/gstruct"
9+
v1 "github.com/prometheus/client_golang/api/prometheus/v1"
10+
"github.com/prometheus/common/model"
11+
"github.com/smartcontractkit/integrations-framework/chaos"
12+
"github.com/smartcontractkit/integrations-framework/chaos/experiments"
613
"github.com/smartcontractkit/integrations-framework/environment"
14+
"github.com/smartcontractkit/integrations-framework/suite/steps"
715
"github.com/smartcontractkit/integrations-framework/suite/testcommon"
816
)
917

10-
var _ = Describe("Alerts suite", func() {
11-
Describe("Alerts", func() {
12-
It("Deploys the alerts stack up to Prometheus", func() {
13-
i := &testcommon.OCRSetupInputs{}
14-
testcommon.DeployOCRForEnv(i, "basic-chainlink", environment.NewChainlinkClusterForAlertsTesting(5))
15-
testcommon.SetupOCRTest(i)
16-
testcommon.CheckRound(i)
17-
testcommon.WriteDataForOTPEToInitializerFileForMockserver(i)
18+
var _ = Describe("OCR Alerts suite", func() {
19+
var (
20+
testSetup *testcommon.OCRSetupInputs
21+
)
1822

19-
err := i.SuiteSetup.Env.DeploySpecs(environment.OtpeGroup())
23+
BeforeEach(func() {
24+
By("Deploying the basic environment", func() {
25+
testSetup = &testcommon.OCRSetupInputs{}
26+
testcommon.DeployOCRForEnv(
27+
testSetup,
28+
"basic-chainlink",
29+
environment.NewChainlinkClusterForAlertsTesting(6),
30+
)
31+
testcommon.SetupOCRTest(testSetup)
32+
})
33+
34+
By("Creating initializer file for mockserver", func() {
35+
steps.WriteDataForOTPEToInitializerFileForMockserver(
36+
testSetup.OCRInstance.Address(),
37+
testSetup.ChainlinkNodes,
38+
)
39+
})
40+
41+
By("Deploying otpe and prometheus", func() {
42+
err := testSetup.SuiteSetup.Env.DeploySpecs(environment.OtpeGroup())
43+
Expect(err).ShouldNot(HaveOccurred())
44+
45+
err = testSetup.SuiteSetup.Env.DeploySpecs(environment.PrometheusGroup())
2046
Expect(err).ShouldNot(HaveOccurred())
47+
})
48+
})
49+
50+
Describe("Telemetry Down Alerts", func() {
51+
It("Doesn't start the OCR protocol", func() {
52+
prometheus, err := environment.GetPrometheusClientFromEnv(testSetup.SuiteSetup.Env)
53+
Expect(err).ShouldNot(HaveOccurred())
54+
55+
Eventually(func(g Gomega) []v1.Alert {
56+
alerts, err := prometheus.Alerts(context.Background())
57+
g.Expect(err).ShouldNot(HaveOccurred())
58+
return alerts.Alerts
59+
}, "5m", "15s").Should(ContainElement(MatchFields(IgnoreExtras, Fields{
60+
"Labels": MatchKeys(IgnoreExtras, Keys{
61+
model.LabelName("alertname"): Equal(model.LabelValue("Telemetry Down (infra)")),
62+
}),
63+
"State": Equal(v1.AlertState("firing")),
64+
})))
65+
})
66+
67+
It("Shuts down all chainlink nodes after some successful rounds", func() {
68+
testcommon.SendOCRJobs(testSetup)
69+
testcommon.CheckRound(testSetup)
70+
71+
experimentSpecs := make([]chaos.Experimentable, len(testSetup.ChainlinkNodes[1:]))
72+
73+
for i := 0; i < len(testSetup.ChainlinkNodes[1:]); i++ {
74+
experimentSpecs[i] = &experiments.PodKill{
75+
TargetAppLabel: fmt.Sprintf("chainlink-%d", i+1),
76+
}
77+
}
78+
79+
for _, experimentSpec := range experimentSpecs {
80+
_, err := testSetup.SuiteSetup.Env.ApplyChaos(experimentSpec)
81+
Expect(err).ShouldNot(HaveOccurred())
82+
}
83+
84+
prometheus, err := environment.GetPrometheusClientFromEnv(testSetup.SuiteSetup.Env)
85+
Expect(err).ShouldNot(HaveOccurred())
86+
87+
Eventually(func(g Gomega) []v1.Alert {
88+
alerts, err := prometheus.Alerts(context.Background())
89+
g.Expect(err).ShouldNot(HaveOccurred())
90+
return alerts.Alerts
91+
}, "5m", "15s").Should(ContainElement(MatchFields(IgnoreExtras, Fields{
92+
"Labels": MatchKeys(IgnoreExtras, Keys{
93+
model.LabelName("alertname"): Equal(model.LabelValue("Telemetry Down (infra)")),
94+
}),
95+
"State": Equal(v1.AlertState("firing")),
96+
})))
97+
})
98+
})
2199

22-
err = i.SuiteSetup.Env.DeploySpecs(environment.PrometheusGroup())
100+
AfterEach(func() {
101+
By("Stop chaos", func() {
102+
err := testSetup.SuiteSetup.Env.StopAllChaos()
23103
Expect(err).ShouldNot(HaveOccurred())
24104
})
105+
By("Tearing down the environment", testSetup.SuiteSetup.TearDown())
25106
})
26107
})

‎suite/chaos/chaos_ocr_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ var _ = XDescribeTable("OCR chaos tests @chaos-ocr", func(
2020
Context("Runs OCR test with a chaos modifier", func() {
2121
testcommon.DeployOCRForEnv(i, envName, envInit)
2222
testcommon.SetupOCRTest(i)
23+
testcommon.SendOCRJobs(i)
2324
_, err := i.SuiteSetup.Env.ApplyChaos(chaosSpec)
2425
Expect(err).ShouldNot(HaveOccurred())
2526
testcommon.CheckRound(i)
@@ -34,7 +35,7 @@ var _ = XDescribeTable("OCR chaos tests @chaos-ocr", func(
3435
},
3536
Entry("One node pod failure",
3637
"basic-chainlink",
37-
environment.NewChainlinkCluster(5),
38+
environment.NewChainlinkCluster(6),
3839
&experiments.PodFailure{
3940
LabelKey: "app",
4041
LabelValue: "chainlink-0",

‎suite/contracts/contracts_ocr_test.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,14 @@ var _ = Describe("OCR Feed @ocr", func() {
1616
i := &testcommon.OCRSetupInputs{}
1717
testcommon.DeployOCRForEnv(i, envName, envInit)
1818
testcommon.SetupOCRTest(i)
19+
testcommon.SendOCRJobs(i)
1920
testcommon.CheckRound(i)
2021
By("Printing gas stats", func() {
2122
i.SuiteSetup.Client.GasStats().PrintStats()
2223
})
2324
By("Tearing down the environment", i.SuiteSetup.TearDown())
2425
},
25-
Entry("all the same version", "basic-chainlink", environment.NewChainlinkCluster(5)),
26-
Entry("different versions", "mixed-version-chainlink", environment.NewMixedVersionChainlinkCluster(5, 2)),
26+
Entry("all the same version", "basic-chainlink", environment.NewChainlinkCluster(6)),
27+
Entry("different versions", "mixed-version-chainlink", environment.NewMixedVersionChainlinkCluster(6, 2)),
2728
)
2829
})

‎suite/steps/alerts_steps.go‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
package steps
2+
3+
import (
4+
"encoding/json"
5+
. "github.com/onsi/gomega"
6+
"github.com/smartcontractkit/integrations-framework/client"
7+
"github.com/smartcontractkit/integrations-framework/environment/charts/mockserver"
8+
"github.com/smartcontractkit/integrations-framework/tools"
9+
"os"
10+
"path/filepath"
11+
)
12+
13+
// WriteDataForOTPEToInitializerFileForMockserver Write to initializerJson mocked weiwatchers data needed for otpe
14+
func WriteDataForOTPEToInitializerFileForMockserver(offChainAggregatorInstanceAddress string, chainlinkNodes []client.Chainlink) {
15+
contractInfo := mockserver.ContractInfoJSON{
16+
ContractVersion: 4,
17+
Path: "test",
18+
Status: "live",
19+
ContractAddress: offChainAggregatorInstanceAddress,
20+
}
21+
22+
contractsInfo := []mockserver.ContractInfoJSON{contractInfo}
23+
24+
contractsInitializer := mockserver.HttpInitializer{
25+
Request: mockserver.HttpRequest{Path: "/contracts.json"},
26+
Response: mockserver.HttpResponse{Body: contractsInfo},
27+
}
28+
29+
var nodesInfo []mockserver.NodeInfoJSON
30+
31+
for _, chainlink := range chainlinkNodes {
32+
ocrKeys, err := chainlink.ReadOCRKeys()
33+
Expect(err).ShouldNot(HaveOccurred())
34+
nodeInfo := mockserver.NodeInfoJSON{
35+
NodeAddress: []string{ocrKeys.Data[0].Attributes.OnChainSigningAddress},
36+
ID: ocrKeys.Data[0].ID,
37+
}
38+
nodesInfo = append(nodesInfo, nodeInfo)
39+
}
40+
41+
nodesInitializer := mockserver.HttpInitializer{
42+
Request: mockserver.HttpRequest{Path: "/nodes.json"},
43+
Response: mockserver.HttpResponse{Body: nodesInfo},
44+
}
45+
initializers := []mockserver.HttpInitializer{contractsInitializer, nodesInitializer}
46+
47+
initializersBytes, err := json.Marshal(initializers)
48+
Expect(err).ShouldNot(HaveOccurred())
49+
50+
fileName := filepath.Join(tools.ProjectRoot, "environment/charts/mockserver-config/static/initializerJson.json")
51+
f, err := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755)
52+
Expect(err).ShouldNot(HaveOccurred())
53+
54+
body := string(initializersBytes)
55+
_, err = f.WriteString(body)
56+
Expect(err).ShouldNot(HaveOccurred())
57+
58+
err = f.Close()
59+
Expect(err).ShouldNot(HaveOccurred())
60+
}

‎suite/testcommon/ocr.go‎

Lines changed: 6 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,8 @@ package testcommon
22

33
import (
44
"context"
5-
"encoding/json"
65
"fmt"
7-
"github.com/smartcontractkit/integrations-framework/environment/charts/mockserver"
86
"math/big"
9-
"os"
10-
"path/filepath"
117
"time"
128

139
. "github.com/onsi/ginkgo"
@@ -70,16 +66,19 @@ func SetupOCRTest(i *OCRSetupInputs) {
7066
Expect(err).ShouldNot(HaveOccurred())
7167
err = i.OCRInstance.SetConfig(
7268
i.DefaultWallet,
73-
i.ChainlinkNodes,
74-
contracts.DefaultOffChainAggregatorConfig(len(i.ChainlinkNodes)),
69+
i.ChainlinkNodes[1:],
70+
contracts.DefaultOffChainAggregatorConfig(len(i.ChainlinkNodes[1:])),
7571
)
7672
Expect(err).ShouldNot(HaveOccurred())
7773
err = i.OCRInstance.Fund(i.DefaultWallet, nil, big.NewFloat(2))
7874
Expect(err).ShouldNot(HaveOccurred())
7975
err = i.SuiteSetup.Client.WaitForEvents()
8076
Expect(err).ShouldNot(HaveOccurred())
8177
})
78+
}
8279

80+
// SendOCRJobs sends ocr jobs to chainlink nodes
81+
func SendOCRJobs(i *OCRSetupInputs) {
8382
By("Sending OCR jobs to chainlink nodes", func() {
8483
// Initialize bootstrap node
8584
bootstrapNode := i.ChainlinkNodes[0]
@@ -165,53 +164,4 @@ func CheckRound(i *OCRSetupInputs) {
165164
Expect(err).ShouldNot(HaveOccurred())
166165
Expect(answer.Int64()).Should(Equal(int64(10)), "Latest answer from OCR is not as expected")
167166
})
168-
}
169-
170-
// WriteDataForOTPEToInitializerFileForMockserver Write to initializerJson mocked weiwatchers data needed for otpe
171-
func WriteDataForOTPEToInitializerFileForMockserver(i *OCRSetupInputs) {
172-
contractInfo := mockserver.ContractInfoJSON{
173-
ContractVersion: 4,
174-
Path: "test",
175-
Status: "live",
176-
ContractAddress: i.OCRInstance.Address(),
177-
}
178-
179-
contractsInfo := []mockserver.ContractInfoJSON{contractInfo}
180-
181-
contractsInitializer := mockserver.HttpInitializer{
182-
Request: mockserver.HttpRequest{Path: "/contracts.json"},
183-
Response: mockserver.HttpResponse{Body: contractsInfo},
184-
}
185-
186-
var nodesInfo []mockserver.NodeInfoJSON
187-
188-
for _, chainlink := range i.ChainlinkNodes {
189-
ocrKeys, err := chainlink.ReadOCRKeys()
190-
Expect(err).ShouldNot(HaveOccurred())
191-
nodeInfo := mockserver.NodeInfoJSON{
192-
NodeAddress: []string{ocrKeys.Data[0].Attributes.OnChainSigningAddress},
193-
ID: ocrKeys.Data[0].ID,
194-
}
195-
nodesInfo = append(nodesInfo, nodeInfo)
196-
}
197-
198-
nodesInitializer := mockserver.HttpInitializer{
199-
Request: mockserver.HttpRequest{Path: "/nodes.json"},
200-
Response: mockserver.HttpResponse{Body: nodesInfo},
201-
}
202-
initializers := []mockserver.HttpInitializer{contractsInitializer, nodesInitializer}
203-
204-
initializersBytes, err := json.Marshal(initializers)
205-
Expect(err).ShouldNot(HaveOccurred())
206-
207-
fileName := filepath.Join(tools.ProjectRoot, "environment/charts/mockserver-config/static/initializerJson.json")
208-
f, err := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755)
209-
Expect(err).ShouldNot(HaveOccurred())
210-
211-
body := string(initializersBytes)
212-
_, err = f.WriteString(body)
213-
Expect(err).ShouldNot(HaveOccurred())
214-
215-
err = f.Close()
216-
Expect(err).ShouldNot(HaveOccurred())
217-
}
167+
}

0 commit comments

Comments
 (0)