Skip to content

Commit aa454c9

Browse files
Merge pull request #384 from RaphaelBut/fix-generate
Simplify local testing steps
2 parents c94b875 + f734511 commit aa454c9

File tree

2 files changed

+26
-38
lines changed

2 files changed

+26
-38
lines changed

README.md

Lines changed: 22 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,6 @@ To add a new alert investigation:
7676
> **Note:** When writing an investiation, you can use them right away.
7777
They are initialized for you and passed to the investigation via investigation.Resources.
7878

79-
8079
* [AWS](https://github.com/aws/aws-sdk-go) -- Logging into the cluster, retreiving instance info and AWS CloudTrail events.
8180
- See `pkg/aws`
8281
* [PagerDuty](https://github.com/PagerDuty/go-pagerduty) -- Retrieving alert info, esclating or silencing incidents, and adding notes.
@@ -90,24 +89,28 @@ They are initialized for you and passed to the investigation via investigation.R
9089

9190
## Testing locally
9291

93-
### Pre-requirements
94-
- an existing cluster
95-
- an existing PagerDuty incident for the cluster and alert type that is being tested
96-
97-
To quickly create an incident for a cluster_id, you can run `./test/generate_incident.sh <alertname> <clusterid>`.
98-
Example usage:`./test/generate_incident.sh ClusterHasGoneMissing 2b94brrrrrrrrrrrrrrrrrrhkaj`.
99-
100-
### Running cadctl for an incident ID
101-
1) Export the required ENV variables, see [required ENV variables](#required-env-variables).
102-
2) Create a payload file containing the incident ID
103-
```bash
104-
export INCIDENT_ID=
105-
echo '{"__pd_metadata":{"incident":{"id":"'${INCIDENT_ID}'"}}}' > ./payload
106-
```
107-
3) Run `cadctl` using the payload file
108-
```bash
109-
./bin/cadctl investigate --payload-path payload
110-
```
92+
Requires an existing cluster.
93+
94+
1. Create a test incident and payload file for your cluster
95+
96+
```bash
97+
./test/generate_incident.sh <alertname> <clusterid>
98+
```
99+
100+
2. Export the required env variables from vault
101+
102+
> **Note:** For information on the envs see [required env variables](#required-env-variables).
103+
104+
```
105+
source test/set_stage_env.sh
106+
```
107+
108+
3. `make build`
109+
4. Run `cadctl` with the payload file created by `test/generate_incident.sh`
110+
111+
```bash
112+
./bin/cadctl investigate --payload-path payload
113+
```
111114

112115
### Logging levels
113116

test/generate_incident.sh

Lines changed: 4 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,6 @@
11
#!/bin/bash
22
set -e
33

4-
# Check if pd_token variable is set
5-
if [ -z "$pd_token" ]; then
6-
echo "Warning: pd_token is not set. For more information, see https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTUx-authentication."
7-
echo "The pd_token is needed to automatically retrieve the incident after creating it."
8-
echo "You can still search in the service directory for the created incident, but setting the pd_token ENV in the future will facilitate getting the created incident id."
9-
echo
10-
echo
11-
fi
12-
134
# Define the mapping of alert names to titles
145
# Add more mappings as needed: for the standard service, we should not need to go by title but by the `alertname` field instead.
156
declare -A alert_mapping=(
@@ -25,7 +16,7 @@ print_help() {
2516
for alert_name in "${!alert_mapping[@]}"; do
2617
echo -n "$alert_name, "
2718
done
28-
echo
19+
echo
2920
}
3021
# Check if the correct number of arguments is provided
3122
if [ "$#" -ne 2 ]; then
@@ -49,9 +40,9 @@ alert_title="${alert_mapping[$alert_name]}"
4940
# Load testing routing key and test service url from vault
5041
export VAULT_ADDR="https://vault.devshift.net"
5142
export VAULT_TOKEN="$(vault login -method=oidc -token-only)"
52-
for v in $(vault kv get -format=json osd-sre/configuration-anomaly-detection/cad-testing | jq -r ".data.data|to_entries|map(\"\(.key)=\(.value|tostring)\")|.[]"); do export $v; done
43+
for v in $(vault kv get -format=json osd-sre/configuration-anomaly-detection/cad-testing | jq -r ".data.data|to_entries|map(\"\(.key)=\(.value|tostring)\")|.[]"); do export $v; done
5344
unset VAULT_ADDR VAULT_TOKEN
54-
echo
45+
echo
5546

5647
dedup_key=$(uuidgen)
5748

@@ -82,20 +73,14 @@ if [[ $response != *"Event processed"* ]]; then
8273
fi
8374
echo
8475

85-
# Point to the service directory if we can't auto retrieve the incident via the pd token
86-
if [ -z "$pd_token" ]; then
87-
echo "Incident successfully created. See ${pd_test_service_url} to retrieve the incident ID."
88-
exit 0
89-
fi
90-
9176
# Pagerduty seems to need a short while to create the incident
9277
# Added this as we intermittently fail to get the incident id otherwise
9378
sleep 2
9479

9580
INCIDENT_ID=$(curl --silent --request GET \
9681
--url "https://api.pagerduty.com/incidents?incident_key=${dedup_key}" \
9782
--header 'Accept: application/json' \
98-
--header "Authorization: Token token=${pd_token}" \
83+
--header "Authorization: Token token=${pd_test_token}" \
9984
--header 'Content-Type: application/json' | jq -r '.incidents[0].id')
10085
echo $INCIDENT_ID
10186
echo '{"__pd_metadata":{"incident":{"id":"'$INCIDENT_ID'"}}}' > ./payload

0 commit comments

Comments
 (0)