Skip to content

Commit 341c92c

Browse files
committed
Add Splunk tests
1 parent 9952804 commit 341c92c

File tree

9 files changed

+3107
-29
lines changed

9 files changed

+3107
-29
lines changed

tools/flakeguard/cmd/aggregate_results.go

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ var AggregateResultsCmd = &cobra.Command{
3232
reportID, _ := cmd.Flags().GetString("report-id")
3333
splunkURL, _ := cmd.Flags().GetString("splunk-url")
3434
splunkToken, _ := cmd.Flags().GetString("splunk-token")
35+
splunkEvent, _ := cmd.Flags().GetString("splunk-event")
3536

3637
// Ensure the output directory exists
3738
if err := fs.MkdirAll(outputDir, 0755); err != nil {
@@ -44,7 +45,11 @@ var AggregateResultsCmd = &cobra.Command{
4445
s.Start()
4546

4647
// Load test reports from JSON files and aggregate them
47-
aggregatedReport, err := reports.LoadAndAggregate(resultsPath, reports.WithReportID(reportID), reports.WithSplunk(splunkURL, splunkToken))
48+
aggregatedReport, err := reports.LoadAndAggregate(
49+
resultsPath,
50+
reports.WithReportID(reportID),
51+
reports.WithSplunk(splunkURL, splunkToken, reports.SplunkEvent(splunkEvent)),
52+
)
4853
if err != nil {
4954
s.Stop()
5055
fmt.Println()
@@ -190,6 +195,7 @@ func init() {
190195
AggregateResultsCmd.Flags().String("report-id", "", "Optional identifier for the test report. Will be generated if not provided")
191196
AggregateResultsCmd.Flags().String("splunk-url", "", "Optional url to simultaneously send the test results to splunk")
192197
AggregateResultsCmd.Flags().String("splunk-token", "", "Optional Splunk HEC token to simultaneously send the test results to splunk")
198+
AggregateResultsCmd.Flags().String("splunk-event", "manual", "Optional Splunk event to send as the triggering event for the test results")
193199

194200
if err := AggregateResultsCmd.MarkFlagRequired("results-path"); err != nil {
195201
log.Fatal().Err(err).Msg("Error marking flag as required")

tools/flakeguard/go.mod

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,13 @@ go 1.21.9
44

55
require (
66
github.com/briandowns/spinner v1.23.1
7+
github.com/go-resty/resty/v2 v2.16.2
78
github.com/google/go-github/v67 v67.0.0
89
github.com/google/uuid v1.6.0
910
github.com/rs/zerolog v1.33.0
1011
github.com/spf13/cobra v1.8.1
1112
golang.org/x/oauth2 v0.24.0
13+
golang.org/x/sync v0.9.0
1214
golang.org/x/text v0.20.0
1315
)
1416

@@ -19,8 +21,9 @@ require (
1921
github.com/mattn/go-colorable v0.1.13 // indirect
2022
github.com/mattn/go-isatty v0.0.19 // indirect
2123
github.com/pmezard/go-difflib v1.0.0 // indirect
22-
golang.org/x/sys v0.12.0 // indirect
23-
golang.org/x/term v0.1.0 // indirect
24+
golang.org/x/net v0.27.0 // indirect
25+
golang.org/x/sys v0.22.0 // indirect
26+
golang.org/x/term v0.22.0 // indirect
2427
gopkg.in/yaml.v3 v3.0.1 // indirect
2528
)
2629

tools/flakeguard/go.sum

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
66
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
77
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
88
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
9+
github.com/go-resty/resty/v2 v2.16.2 h1:CpRqTjIzq/rweXUt9+GxzzQdlkqMdt8Lm/fuK/CAbAg=
10+
github.com/go-resty/resty/v2 v2.16.2/go.mod h1:0fHAoK7JoBy/Ch36N8VFeMsK7xQOHhvWaC3iOktwmIU=
911
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
1012
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
1113
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
@@ -36,16 +38,23 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
3638
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
3739
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
3840
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
41+
golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
42+
golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
3943
golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE=
4044
golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
45+
golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ=
46+
golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
4147
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
4248
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
43-
golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
4449
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
45-
golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw=
46-
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
50+
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
51+
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
52+
golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk=
53+
golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
4754
golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=
4855
golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=
56+
golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U=
57+
golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
4958
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
5059
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
5160
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

tools/flakeguard/reports/data.go

Lines changed: 91 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,15 @@ import (
66
"sort"
77
"strings"
88
"time"
9+
10+
"github.com/go-resty/resty/v2"
11+
"github.com/rs/zerolog/log"
12+
"golang.org/x/sync/errgroup"
913
)
1014

1115
// TestReport reports on the parameters and results of one to many test runs
1216
type TestReport struct {
13-
ReportID string `json:"report_id"`
17+
ID string `json:"id"`
1418
GoProject string `json:"go_project"`
1519
HeadSHA string `json:"head_sha"`
1620
BaseSHA string `json:"base_sha"`
@@ -25,6 +29,9 @@ type TestReport struct {
2529

2630
// TestResult contains the results and outputs of a single test
2731
type TestResult struct {
32+
// ReportID is the ID of the report this test result belongs to
33+
// used mostly for Splunk logging
34+
ReportID string `json:"report_id"`
2835
TestName string `json:"test_name"`
2936
TestPackage string `json:"test_package"`
3037
PackagePanic bool `json:"package_panic"`
@@ -79,7 +86,7 @@ const (
7986

8087
// SplunkTestReport is the full wrapper structure sent to Splunk for the full test report (sans results)
8188
type SplunkTestReport struct {
82-
Event SplunkTestResultEvent `json:"event"` // https://docs.splunk.com/Splexicon:Event
89+
Event SplunkTestReportEvent `json:"event"` // https://docs.splunk.com/Splexicon:Event
8390
SourceType string `json:"sourcetype"` // https://docs.splunk.com/Splexicon:Sourcetype
8491
}
8592

@@ -180,7 +187,7 @@ func aggregate(reportChan <-chan *TestReport, errChan <-chan error, opts *aggreg
180187
excludedTests := map[string]struct{}{}
181188
selectedTests := map[string]struct{}{}
182189

183-
fullReport.ReportID = opts.reportID
190+
fullReport.ID = opts.reportID
184191
for report := range reportChan {
185192
if fullReport.GoProject == "" {
186193
fullReport.GoProject = report.GoProject
@@ -196,6 +203,7 @@ func aggregate(reportChan <-chan *TestReport, errChan <-chan error, opts *aggreg
196203
selectedTests[test] = struct{}{}
197204
}
198205
for _, result := range report.Results {
206+
result.ReportID = opts.reportID
199207
key := result.TestName + "|" + result.TestPackage
200208
if existing, found := testMap[key]; found {
201209
existing = mergeTestResults(existing, result)
@@ -218,18 +226,97 @@ func aggregate(reportChan <-chan *TestReport, errChan <-chan error, opts *aggreg
218226
fullReport.SelectedTests = append(fullReport.SelectedTests, test)
219227
}
220228

229+
// Send report to Splunk before adding test results
230+
if opts.splunkURL != "" {
231+
err := sendReportToSplunk(opts.splunkURL, opts.splunkToken, opts.splunkEvent, fullReport)
232+
if err != nil {
233+
log.Error().Err(err).Msg("Error sending report to Splunk")
234+
} else {
235+
log.Debug().Str("event", string(opts.splunkEvent)).Msg("Report sent to Splunk")
236+
}
237+
}
238+
221239
// Prepare final results
240+
eg := errgroup.Group{}
222241
var aggregatedResults []TestResult
223-
for _, result := range testMap {
242+
for _, r := range testMap {
243+
result := r
224244
aggregatedResults = append(aggregatedResults, result)
245+
if opts.splunkURL != "" {
246+
eg.Go(func() error {
247+
return sendResultsToSplunk(opts.splunkURL, opts.splunkToken, opts.splunkEvent, result)
248+
})
249+
}
225250
}
226251

227252
sortTestResults(aggregatedResults)
228253
fullReport.Results = aggregatedResults
229254

255+
if splunkErr := eg.Wait(); splunkErr != nil {
256+
log.Error().Err(splunkErr).Msg("Error sending results to Splunk")
257+
} else {
258+
log.Debug().Str("event", string(opts.splunkEvent)).Msg("All results sent to Splunk")
259+
}
230260
return fullReport, nil
231261
}
232262

263+
// sendReportToSplunk sends meta test report data to Splunk
264+
func sendReportToSplunk(url, token string, event SplunkEvent, report *TestReport) error {
265+
client := resty.New()
266+
client.AddRetryAfterErrorCondition().SetRetryCount(3).SetRetryWaitTime(5 * time.Second)
267+
resp, err := client.R().
268+
SetHeader("Authorization", fmt.Sprintf("Splunk %s", token)).
269+
SetHeader("Content-Type", "application/json").
270+
SetBody(SplunkTestReport{
271+
Event: SplunkTestReportEvent{
272+
Event: event,
273+
Type: Report,
274+
Data: *report,
275+
},
276+
SourceType: "flakeguard_json",
277+
}).
278+
Post(url)
279+
if err != nil {
280+
return err
281+
}
282+
if resp.IsError() {
283+
return fmt.Errorf("error sending report to Splunk: %s", resp.String())
284+
}
285+
return nil
286+
}
287+
288+
func sendResultsToSplunk(url, token string, event SplunkEvent, results ...TestResult) error {
289+
client := resty.New()
290+
client.AddRetryAfterErrorCondition().SetRetryCount(3).SetRetryWaitTime(5 * time.Second)
291+
eg := errgroup.Group{}
292+
for _, r := range results {
293+
result := r
294+
eg.Go(func() error {
295+
resp, err := client.R().
296+
SetHeader("Authorization", fmt.Sprintf("Splunk %s", token)).
297+
SetHeader("Content-Type", "application/json").
298+
SetBody(SplunkTestResult{
299+
Event: SplunkTestResultEvent{
300+
Event: event,
301+
Type: Result,
302+
Data: result,
303+
},
304+
SourceType: "flakeguard_json",
305+
}).
306+
Post(url)
307+
if err != nil {
308+
return err
309+
}
310+
if resp.IsError() {
311+
return fmt.Errorf("error sending result to Splunk: %s", resp.String())
312+
}
313+
return nil
314+
})
315+
}
316+
317+
return eg.Wait()
318+
}
319+
233320
func aggregateFromReports(opts *aggregateOptions, reports ...*TestReport) (*TestReport, error) {
234321
reportChan := make(chan *TestReport, len(reports))
235322
errChan := make(chan error, 1)

tools/flakeguard/reports/io.go

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ type aggregateOptions struct {
3737
reportID string
3838
splunkURL string
3939
splunkToken string
40+
splunkEvent SplunkEvent
4041
}
4142

4243
// AggregateOption is a functional option for configuring the aggregation process.
@@ -50,10 +51,11 @@ func WithReportID(reportID string) AggregateOption {
5051
}
5152

5253
// WithSplunk also sends the aggregation to a Splunk instance as events.
53-
func WithSplunk(splunkURL, splunkToken string) AggregateOption {
54+
func WithSplunk(url, token string, event SplunkEvent) AggregateOption {
5455
return func(opts *aggregateOptions) {
55-
opts.splunkURL = splunkURL
56-
opts.splunkToken = splunkToken
56+
opts.splunkURL = url
57+
opts.splunkToken = token
58+
opts.splunkEvent = event
5759
}
5860
}
5961

@@ -192,9 +194,9 @@ func decodeField(decoder *json.Decoder, report *TestReport) error {
192194
if err := decoder.Decode(&report.SelectedTests); err != nil {
193195
return fmt.Errorf("error decoding SelectedTests: %w", err)
194196
}
195-
case "report_id":
196-
if err := decoder.Decode(&report.ReportID); err != nil {
197-
return fmt.Errorf("error decoding ReportID: %w", err)
197+
case "id":
198+
if err := decoder.Decode(&report.ID); err != nil {
199+
return fmt.Errorf("error decoding ID: %w", err)
198200
}
199201
case "results":
200202
token, err := decoder.Token() // Read opening bracket '['
@@ -221,7 +223,6 @@ func decodeField(decoder *json.Decoder, report *TestReport) error {
221223
}
222224
log.Warn().Str("field", fieldName).Msg("Skipped unknown field, check the test report struct to see if it's been properly updated")
223225
}
224-
fmt.Printf("Decoded field: %s\n", fieldName)
225226
return nil
226227
}
227228

Lines changed: 67 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,80 @@
11
package reports
22

33
import (
4+
"encoding/json"
5+
"fmt"
6+
"io"
47
"net/http"
58
"net/http/httptest"
69
"testing"
10+
11+
"github.com/stretchr/testify/assert"
12+
"github.com/stretchr/testify/require"
13+
)
14+
15+
const (
16+
splunkToken = "test-token"
17+
numberReports = 3
18+
reportID = "123"
19+
testsRun = 15
720
)
821

9-
func TestSendSplunkEvents(t *testing.T) {
10-
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
11-
// assert.Equal(t, "application/json", r.Header.Get("Content-Type"))
22+
func TestAggregateResultsSplunk(t *testing.T) {
23+
srv := splunkServer(t)
24+
t.Cleanup(srv.Close)
25+
26+
report, err := LoadAndAggregate("./testdata", WithReportID(reportID), WithSplunk(srv.URL, splunkToken, "test"))
27+
require.NoError(t, err, "LoadAndAggregate failed")
28+
require.NotNil(t, report, "report is nil")
29+
}
30+
31+
func TestAggregateResults(t *testing.T) {
32+
report, err := LoadAndAggregate("./testdata", WithReportID(reportID))
33+
require.NoError(t, err, "LoadAndAggregate failed")
34+
require.NotNil(t, report, "report is nil")
35+
assert.Equal(t, reportID, report.ID, "report ID mismatch")
36+
assert.Equal(t, testsRun, len(report.Results), "report results count mismatch")
37+
assert.Equal(t, testsRun, report.TestRunCount, "report test run count mismatch")
38+
}
39+
40+
func splunkServer(t *testing.T) *httptest.Server {
41+
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
42+
assert.Equal(t, "application/json", r.Header.Get("Content-Type"))
43+
assert.Equal(t, fmt.Sprintf("Splunk %s", splunkToken), r.Header.Get("Authorization"))
44+
45+
// Figure out what the payload is
46+
bodyBytes, err := io.ReadAll(r.Body)
47+
require.NoError(t, err)
48+
defer r.Body.Close()
1249

13-
// body, err := io.ReadAll(r.Body)
14-
// assert.NoError(t, err)
15-
// defer r.Body.Close()
50+
var payload map[string]any
51+
err = json.Unmarshal(bodyBytes, &payload)
52+
require.NoError(t, err, "error parsing splunk event data")
53+
require.NotNil(t, payload, "error parsing splunk event data")
54+
require.NotNil(t, payload["event"], "unable to find event while parsing splunk data")
55+
event := payload["event"].(map[string]any)
56+
require.NotNil(t, event, "error parsing splunk event data")
57+
require.NotNil(t, event["type"], "unable to find inner event type while parsing splunk data")
58+
eventType := event["type"].(string)
59+
require.NotNil(t, eventType, "error parsing splunk event type")
1660

17-
// var receivedEvents []Event
18-
// err = json.Unmarshal(body, &receivedEvents)
19-
// assert.NoError(t, err)
61+
if eventType == string(Report) {
62+
var report TestReport
63+
err := json.Unmarshal(bodyBytes, &report)
64+
require.NoError(t, err, "error parsing report data")
65+
require.NotNil(t, report, "error parsing report data")
66+
assert.Equal(t, reportID, report.ID, "report ID mismatch")
67+
assert.Equal(t, testsRun, len(report.Results), "report results count mismatch")
68+
assert.Equal(t, testsRun, report.TestRunCount, "report test run count mismatch")
69+
} else if eventType == string(Result) {
70+
var result SplunkTestResult
71+
err := json.Unmarshal(bodyBytes, &result)
72+
require.NoError(t, err, "error parsing results data")
73+
require.NotNil(t, result, "error parsing results data")
74+
} else {
75+
t.Errorf("unexpected event type: %s", eventType)
76+
}
2077

21-
// assert.Equal(t, expectedEvents, receivedEvents)
22-
// w.WriteHeader(http.StatusOK)
78+
w.WriteHeader(http.StatusOK)
2379
}))
24-
defer srv.Close()
2580
}

0 commit comments

Comments
 (0)