Skip to content

Commit 5b3a227

Browse files
laurenkrugen-navapbclaurenkrugen-navapbc
authored
BCDA-8777: Update alert destinations for workflows and lambdas (#1175)
## 🎫 Ticket https://jira.cms.gov/browse/BCDA-8777 ## 🛠 Changes **Admin Lambdas:** - Updated slack alerts to be triggered in the `handle` function - All status updates will go to operations channel **Workflows**: - Deploy related workflows have had destination channels updated to deploy channel **General**: - Added slack utils package and constants for slack messages to reduce duplicate code ## ℹ️ Context Alerting channels are currently getting status and informational messages that don't pertain to alerts. To better manage alerts and critical infrastructure related monitoring, our alerting channel should be free of messages that do not indicate an issue. Deploy related updates will only be in the deploy channel and admin lambdas will report status updates to the operations channel, including run failures, since those are expected. <!-- If any of the following security implications apply, this PR must not be merged without Stephen Walter's approval. Explain in this section and add @SJWalter11 as a reviewer. - Adds a new software dependency or dependencies. - Modifies or invalidates one or more of our security controls. - Stores or transmits data that was not stored or transmitted before. - Requires additional review of security implications for other reasons. --> ## 🧪 Validation Integration tests are passing, workflows in expected channels. Looking to verify: - lambdas that run on a schedule (non-admin) route failures to the alerts channel but route success to operations - admin lambdas route all statuses to operations - deploy workflows and related workflows route only to deploy channel - `Starting..` messages removed to reduce noise
1 parent 5a5cbd4 commit 5b3a227

File tree

15 files changed

+100
-180
lines changed

15 files changed

+100
-180
lines changed

‎.github/workflows/build-and-package-all.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -135,9 +135,8 @@ jobs:
135135
with:
136136
method: chat.postMessage
137137
token: ${{ secrets.SLACK_BOT_TOKEN }}
138-
# Sends to bcda-alerts
139138
payload: |
140-
channel: "C034CFU945C"
139+
channel: "C03S23MJFJS"
141140
attachments:
142141
- color: danger
143142
text: "FAILURE: Build and Package BCDA/SSAS (run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.run_id }}>)."

‎.github/workflows/build-and-package-bcda.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,8 @@ jobs:
4343
with:
4444
method: chat.postMessage
4545
token: ${{ secrets.SLACK_BOT_TOKEN }}
46-
# Sends to bcda-alerts
4746
payload: |
48-
channel: "C034CFU945C"
47+
channel: "C03S23MJFJS"
4948
attachments:
5049
- color: danger
5150
text: "FAILURE: Build and Package BCDA/Worker (run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.run_id }}>)."

‎.github/workflows/deploy-all.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -335,9 +335,8 @@ jobs:
335335
with:
336336
method: chat.postMessage
337337
token: ${{ secrets.SLACK_BOT_TOKEN }}
338-
# Sends to bcda-alerts
339338
payload: |
340-
channel: "C034CFU945C"
339+
channel: "C03S23MJFJS"
341340
attachments:
342341
- color: danger
343342
text: "FAILURE: Deploy to ${{ env.RELEASE_ENV }} (run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.run_id }}>)."

‎.github/workflows/migrate-db.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,9 +88,8 @@ jobs:
8888
with:
8989
method: chat.postMessage
9090
token: ${{ secrets.SLACK_BOT_TOKEN }}
91-
# Sends to bcda-alerts
9291
payload: |
93-
channel: "C034CFU945C"
92+
channel: "C03S23MJFJS"
9493
attachments:
9594
- color: danger
9695
text: "FAILURE: DB Migrations in ${{ inputs.env }} (run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.run_id }}>)."

‎.github/workflows/tag-and-release.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -102,9 +102,8 @@ jobs:
102102
with:
103103
method: chat.postMessage
104104
token: ${{ secrets.SLACK_BOT_TOKEN }}
105-
# Sends to bcda-alerts
106105
payload: |
107-
channel: "C034CFU945C"
106+
channel: "C03S23MJFJS"
108107
attachments:
109108
- color: danger
110109
text: "FAILURE: Tag and Release BCDA (${{ env.RELEASE_VERSION }}) and OPS (${{ env.OPS_RELEASE_VERSION }}). Run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.run_id }}>"

‎bcda/lambda/admin_aco_deny/main.go‎

Lines changed: 8 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,11 @@ import (
1313
"github.com/slack-go/slack"
1414

1515
bcdaaws "github.com/CMSgov/bcda-app/bcda/aws"
16+
msgr "github.com/CMSgov/bcda-app/bcda/slackmessenger"
1617

1718
log "github.com/sirupsen/logrus"
1819
)
1920

20-
var slackChannel = "C034CFU945C" // #bcda-alerts
21-
2221
type payload struct {
2322
DenyACOIDs []string `json:"deny_aco_ids"`
2423
}
@@ -28,10 +27,6 @@ type awsParams struct {
2827
SlackToken string
2928
}
3029

31-
type Notifier interface {
32-
PostMessageContext(context.Context, string, ...slack.MsgOption) (string, string, error)
33-
}
34-
3530
func main() {
3631
lambda.Start(handler)
3732
}
@@ -65,46 +60,30 @@ func handler(ctx context.Context, event json.RawMessage) error {
6560

6661
slackClient := slack.New(params.SlackToken)
6762

68-
err = handleACODenies(ctx, conn, data, slackClient)
63+
err = handleACODenies(ctx, conn, data)
6964
if err != nil {
65+
msgr.SendSlackMessage(slackClient, msgr.OperationsChannel, fmt.Sprintf("%s: Deny ACO lambda in %s env.", msgr.FailureMsg, os.Getenv("ENV")), msgr.Danger)
66+
7067
log.Errorf("Failed to handle ACO denies: %+v", err)
7168
return err
7269
}
7370

71+
msgr.SendSlackMessage(slackClient, msgr.OperationsChannel, fmt.Sprintf("%s: Deny ACO lambda in %s env.", msgr.SuccessMsg, os.Getenv("ENV")), msgr.Good)
72+
7473
log.Info("Completed ACO Deny administrative task")
7574

7675
return nil
7776
}
7877

79-
func handleACODenies(ctx context.Context, conn PgxConnection, data payload, notifier Notifier) error {
80-
_, _, err := notifier.PostMessageContext(ctx, slackChannel, slack.MsgOptionText(
81-
fmt.Sprintf("Started ACO Deny lambda in %s env.", os.Getenv("ENV")), false),
82-
)
83-
if err != nil {
84-
log.Errorf("Error sending notifier start message: %+v", err)
85-
}
78+
func handleACODenies(ctx context.Context, conn PgxConnection, data payload) error {
8679

87-
err = denyACOs(ctx, conn, data)
80+
err := denyACOs(ctx, conn, data)
8881
if err != nil {
8982
log.Errorf("Error finding and denying ACOs: %+v", err)
9083

91-
_, _, err := notifier.PostMessageContext(ctx, slackChannel, slack.MsgOptionText(
92-
fmt.Sprintf("Failed: ACO Deny List lambda in %s env.", os.Getenv("ENV")), false),
93-
)
94-
if err != nil {
95-
log.Errorf("Error sending notifier failure message: %+v", err)
96-
}
97-
9884
return err
9985
}
10086

101-
_, _, err = notifier.PostMessageContext(ctx, slackChannel, slack.MsgOptionText(
102-
fmt.Sprintf("Success: ACO Deny List lambda in %s env.", os.Getenv("ENV")), false),
103-
)
104-
if err != nil {
105-
log.Errorf("Error sending notifier success message: %+v", err)
106-
}
107-
10887
return nil
10988
}
11089

‎bcda/lambda/admin_aco_deny/main_test.go‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,21 +3,11 @@ package main
33
import (
44
"context"
55
"testing"
6-
"time"
76

87
"github.com/pashagolub/pgxmock/v4"
9-
"github.com/slack-go/slack"
108
"github.com/stretchr/testify/assert"
119
)
1210

13-
type mockNotifier struct {
14-
Notifier
15-
}
16-
17-
func (m *mockNotifier) PostMessageContext(ctx context.Context, channelID string, options ...slack.MsgOption) (string, string, error) {
18-
return channelID, time.Now().String(), nil
19-
}
20-
2111
func TestHandleACODenies(t *testing.T) {
2212
ctx := context.Background()
2313

@@ -29,6 +19,6 @@ func TestHandleACODenies(t *testing.T) {
2919
WithArgs(mockTermination{}, testACODenies).
3020
WillReturnResult(pgxmock.NewResult("UPDATE", 3))
3121

32-
err = handleACODenies(ctx, mockConn, payload{testACODenies}, &mockNotifier{})
22+
err = handleACODenies(ctx, mockConn, payload{testACODenies})
3323
assert.Nil(t, err)
3424
}

‎bcda/lambda/admin_create_aco/main.go‎

Lines changed: 9 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,11 @@ import (
1717
bcdaaws "github.com/CMSgov/bcda-app/bcda/aws"
1818
"github.com/CMSgov/bcda-app/bcda/models"
1919
"github.com/CMSgov/bcda-app/bcda/service"
20+
msgr "github.com/CMSgov/bcda-app/bcda/slackmessenger"
2021

2122
log "github.com/sirupsen/logrus"
2223
)
2324

24-
var slackChannel = "C034CFU945C" // #bcda-alerts
25-
2625
type payload struct {
2726
Name string `json:"name"`
2827
CMSID string `json:"cms_id"`
@@ -34,10 +33,6 @@ type awsParams struct {
3433
slackToken string
3534
}
3635

37-
type Notifier interface {
38-
PostMessageContext(context.Context, string, ...slack.MsgOption) (string, string, error)
39-
}
40-
4136
func main() {
4237
lambda.Start(handler)
4338
}
@@ -73,12 +68,16 @@ func handler(ctx context.Context, event json.RawMessage) error {
7368
id := uuid.NewRandom()
7469

7570
if data.CleanUp == nil {
71+
7672
// run the regular logic (non-rollback transaction)
77-
err = handleCreateACO(ctx, conn, data, id, slackClient)
73+
err = handleCreateACO(ctx, conn, data, id)
7874
if err != nil {
75+
msgr.SendSlackMessage(slackClient, msgr.OperationsChannel, fmt.Sprintf("%s: Create ACO lambda in %s env.", msgr.FailureMsg, os.Getenv("ENV")), msgr.Danger)
7976
log.Errorf("Failed to handle Create ACO: %+v", err)
8077
return err
8178
}
79+
msgr.SendSlackMessage(slackClient, msgr.OperationsChannel, fmt.Sprintf("%s: Create ACO lambda in %s env.", msgr.SuccessMsg, os.Getenv("ENV")), msgr.Good)
80+
8281
} else {
8382
// create a rollbackable transaction
8483
tx, cErr := conn.Begin(ctx)
@@ -87,7 +86,7 @@ func handler(ctx context.Context, event json.RawMessage) error {
8786
return err
8887
}
8988

90-
err = handleCreateACO(ctx, tx, data, id, slackClient)
89+
err = handleCreateACO(ctx, tx, data, id)
9190
if err != nil {
9291
log.Errorf("Failed to handle Create ACO: %+v", err)
9392
return err
@@ -105,13 +104,7 @@ func handler(ctx context.Context, event json.RawMessage) error {
105104
return nil
106105
}
107106

108-
func handleCreateACO(ctx context.Context, conn PgxConnection, data payload, id uuid.UUID, notifier Notifier) error {
109-
_, _, err := notifier.PostMessageContext(ctx, slackChannel, slack.MsgOptionText(
110-
fmt.Sprintf("Started Create ACO lambda in %s env.", os.Getenv("ENV")), false),
111-
)
112-
if err != nil {
113-
log.Errorf("Error sending notifier start message: %+v", err)
114-
}
107+
func handleCreateACO(ctx context.Context, conn PgxConnection, data payload, id uuid.UUID) error {
115108

116109
if data.Name == "" {
117110
return errors.New("ACO name must be provided")
@@ -132,18 +125,11 @@ func handleCreateACO(ctx context.Context, conn PgxConnection, data payload, id u
132125

133126
aco := models.ACO{Name: data.Name, CMSID: cmsIDPt, UUID: id, ClientID: id.String()}
134127

135-
err = createACO(context.Background(), conn, aco)
128+
err := createACO(context.Background(), conn, aco)
136129
if err != nil {
137130
return err
138131
}
139132

140-
_, _, err = notifier.PostMessageContext(ctx, slackChannel, slack.MsgOptionText(
141-
fmt.Sprintf("Success: Create ACO lambda in %s env.", os.Getenv("ENV")), false),
142-
)
143-
if err != nil {
144-
log.Errorf("Error sending notifier success message: %+v", err)
145-
}
146-
147133
return nil
148134
}
149135

‎bcda/lambda/admin_create_aco/main_test.go‎

Lines changed: 5 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,10 @@ import (
44
"context"
55
"errors"
66
"testing"
7-
"time"
87

98
"github.com/jackc/pgx/v5"
109
"github.com/pashagolub/pgxmock/v4"
1110
"github.com/pborman/uuid"
12-
"github.com/slack-go/slack"
1311
"github.com/stretchr/testify/assert"
1412
"github.com/stretchr/testify/suite"
1513
)
@@ -54,14 +52,6 @@ func TestHandleCreateACOTestSuite(t *testing.T) {
5452
suite.Run(t, new(HandleCreateACOTestSuite))
5553
}
5654

57-
type mockNotifier struct {
58-
Notifier
59-
}
60-
61-
func (m *mockNotifier) PostMessageContext(ctx context.Context, channelID string, options ...slack.MsgOption) (string, string, error) {
62-
return channelID, time.Now().String(), nil
63-
}
64-
6555
type mockString struct{}
6656

6757
func (ms mockString) Match(v any) bool {
@@ -98,7 +88,7 @@ func TestHandleCreateACOSuccess(t *testing.T) {
9888
data := payload{"TESTACO", "TEST002", nil}
9989
id := uuid.NewRandom()
10090

101-
err = handleCreateACO(ctx, mockConn, data, id, &mockNotifier{})
91+
err = handleCreateACO(ctx, mockConn, data, id)
10292
assert.Nil(t, err)
10393
}
10494

@@ -124,30 +114,30 @@ func TestHandleCreateACOFailure(t *testing.T) {
124114
data := payload{"TESTACO", "TEST002", nil}
125115
id := uuid.NewRandom()
126116

127-
err = handleCreateACO(ctx, mockConn, data, id, &mockNotifier{})
117+
err = handleCreateACO(ctx, mockConn, data, id)
128118
assert.ErrorContains(t, err, "test error")
129119
}
130120

131121
func (c *HandleCreateACOTestSuite) TestHandleCreateACOInvalidCMSID() {
132122
data := payload{"TESTACO", "12345678", nil}
133123
id := uuid.NewRandom()
134124

135-
err := handleCreateACO(c.ctx, c.tx, data, id, &mockNotifier{})
125+
err := handleCreateACO(c.ctx, c.tx, data, id)
136126
assert.ErrorContains(c.T(), err, "invalid")
137127
}
138128

139129
func (c *HandleCreateACOTestSuite) TestHandleCreateACOMissingName() {
140130
data := payload{"", "TEST510", nil}
141131
id := uuid.NewRandom()
142132

143-
err := handleCreateACO(c.ctx, c.tx, data, id, &mockNotifier{})
133+
err := handleCreateACO(c.ctx, c.tx, data, id)
144134
assert.ErrorContains(c.T(), err, "ACO name must be provided")
145135
}
146136

147137
func (c *HandleCreateACOTestSuite) TestHandleCreateACOMissingCMSID() {
148138
data := payload{"Test ACO 5", "", nil}
149139
id := uuid.NewRandom()
150140

151-
err := handleCreateACO(c.ctx, c.tx, data, id, &mockNotifier{})
141+
err := handleCreateACO(c.ctx, c.tx, data, id)
152142
assert.ErrorContains(c.T(), err, "CMSID must be provided")
153143
}

0 commit comments

Comments
 (0)