diff --git a/README.md b/README.md index 75951d4f0b6..7d22b918ed9 100644 --- a/README.md +++ b/README.md @@ -165,3 +165,12 @@ To generate plots you should execute: $ go run ./cmd/batch --gh-token /path/to/token --source --path $(pwd)/output --mode plot --target-metric merged-prs --start-date 2020-05-19 ``` Plot mode requires data previously generated by fetch mode. + +### html-report command + +A command that generates HTML reports of test case failures per SIG. To create a report for SIG Compute failures: +``` +go run ./cmd/html-report --sig compute --results-path ./output/kubevirt/kubevirt/results.json --path /tmp/ +``` + +This should create a HTML report called sig-compute-failure-report.html under /tmp/. diff --git a/cmd/html-report/main.go b/cmd/html-report/main.go new file mode 100644 index 00000000000..6b77c9daa01 --- /dev/null +++ b/cmd/html-report/main.go @@ -0,0 +1,39 @@ +package main + +import ( + "fmt" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "github.com/kubevirt/ci-health/pkg/constants" + "github.com/kubevirt/ci-health/pkg/htmlreport" + "github.com/kubevirt/ci-health/pkg/types" +) + +func main() { + opt := &types.Options{ + Path: constants.DefaultPath, + Sig: "compute", + ResultsPath: constants.DefaultResultsPath, + } + + cmd := &cobra.Command{ + Run: func(cmd *cobra.Command, arguments []string) { + if err := htmlreport.Generate(opt); err != nil { + log.Fatalf("error generating HTML report: %v", err) + } + reportPath := fmt.Sprintf("%s/sig-%s-failure-report.html", opt.Path, opt.Sig) + log.Printf("HTML report written to %s", reportPath) + }, + } + flag := cmd.Flags() + + flag.StringVar(&opt.Path, "path", opt.Path, "Path to output HTML reports to") + flag.StringVar(&opt.Sig, "sig", opt.Sig, "Name of SIG to generate failure report for") + flag.StringVar(&opt.ResultsPath, "results-path", opt.ResultsPath, "Path to the results.json file created by ci-health") + + if err := cmd.Execute(); err != nil { + log.Fatalf("error: %v", err) + } +} diff --git a/go.mod b/go.mod index 0c66118fad7..834e01b7605 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ toolchain go1.24.3 require ( github.com/avast/retry-go v3.0.0+incompatible + github.com/joshdk/go-junit v1.0.0 github.com/narqo/go-badge v0.0.0-20230821190521-c9a75c019a59 github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.27.10 diff --git a/go.sum b/go.sum index 4b89120f457..8417f1bad54 100644 --- a/go.sum +++ b/go.sum @@ -64,6 +64,8 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= diff --git a/pkg/constants/main.go b/pkg/constants/main.go index 81db6307f13..7472dce5999 100644 --- a/pkg/constants/main.go +++ b/pkg/constants/main.go @@ -18,6 +18,7 @@ const ( SIGRetests = "SIGRetests" DefaultPath = "/tmp/test" + DefaultResultsPath = "" DefaultTokenPath = "" DefaultSource = "kubevirt/kubevirt" DefaultDataDays = 7 diff --git a/pkg/htmlreport/main.go b/pkg/htmlreport/main.go new file mode 100644 index 00000000000..20cc9f40b51 --- /dev/null +++ b/pkg/htmlreport/main.go @@ -0,0 +1,182 @@ +package htmlreport + +import ( + _ "embed" + "encoding/json" + "encoding/xml" + "fmt" + "html/template" + "io" + "net/http" + "os" + "regexp" + "strings" + + "github.com/joshdk/go-junit" + "github.com/kubevirt/ci-health/pkg/sigretests" + "github.com/kubevirt/ci-health/pkg/types" + log "github.com/sirupsen/logrus" +) + +var ( + //go:embed sig-failure-report.gohtml + sigFailureReportTemplate string +) + +type HTMLReportResults struct { + Data struct { + SIGRetests struct { + FailedJobLeaderBoard []types.FailedJob `json:"FailedJobLeaderBoard"` + } `json:"SIGRetests"` + } `json:"Data"` +} + +type Failure struct { + XMLName xml.Name `xml:"failure"` + Message string `xml:"message,attr"` + Type string `xml:"type,attr"` + Value string `xml:",chardata"` +} + +type SigFailure struct { + Sig string + JobName string + FailureURL string + Testcase []junit.Test +} + +var jobRegexAliases = map[string]string{ + "compute": "sig-compute$|sig-compute-serial$|sig-compute-migrations$|sig-operator$|vgpu$|sev$", + "network": "sig-network$|sriov$", + "storage": "sig-storage$", +} + +func fetchResults(resultsPath string) (*HTMLReportResults, error) { + body, err := os.ReadFile(resultsPath) + if err != nil { + return nil, fmt.Errorf("failed to read results.json file: %w", err) + } + + var results HTMLReportResults + if err := json.Unmarshal(body, &results); err != nil { + return nil, fmt.Errorf("failed to unmarshal results.json: %w", err) + } + + return &results, nil +} + +func fetchJunit(url string) ([]junit.Suite, error) { + resp, err := sigretests.HttpGetWithRetry(url) + if err != nil { + return nil, fmt.Errorf("failed to fetch %s: %s", url, err) + } + defer resp.Body.Close() + + // Ignore missing junit files as it suggests an issue with the job + if resp.StatusCode == http.StatusNotFound { + return nil, nil + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to fetch %s: status code %d", url, resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read %s body: %w", url, err) + } + + testsuite, err := junit.Ingest(body) + if err == nil { + return testsuite, nil + } + + return nil, fmt.Errorf("failed to unmarshal junit.functest.xml as or ") +} + +func constructReportFilePath(opt *types.Options) string { + return fmt.Sprintf("%s/sig-%s-failure-report.html", opt.Path, opt.Sig) +} + +func constructJunitURL(failureURL string) string { + junitURL := strings.Replace(failureURL, "prow.ci.kubevirt.io//view/gs", "gcsweb.ci.kubevirt.io/gcs", 1) + if !strings.HasSuffix(junitURL, "/") { + junitURL += "/" + } + junitURL += "artifacts/junit.functest.xml" + return junitURL +} + +func Generate(opt *types.Options) error { + + if opt.ResultsPath == "" { + return fmt.Errorf("the path to results.json is required") + } + + jobRegex, ok := jobRegexAliases[opt.Sig] + if !ok { + return fmt.Errorf("unknown SIG: %s", opt.Sig) + } + + results, err := fetchResults(opt.ResultsPath) + if err != nil { + return fmt.Errorf("failed to parse results.json: %w", err) + } + + compiledRegex, err := regexp.Compile(jobRegex) + if err != nil { + return fmt.Errorf("invalid job regex provided: %w", err) + } + + var sigFailures []SigFailure + + for _, job := range results.Data.SIGRetests.FailedJobLeaderBoard { + if !compiledRegex.MatchString(job.JobName) { + continue + } + for _, failureURL := range job.FailureURLs { + var sigFail SigFailure + junitURL := constructJunitURL(failureURL) + testSuites, err := fetchJunit(junitURL) + if err != nil { + log.Warnf("failed to fetch junit results: %s", err) + continue + } + if testSuites == nil { + // SIG CI failure + continue + } + sigFail.Sig = opt.Sig + sigFail.JobName = job.JobName + sigFail.FailureURL = failureURL + + for _, suite := range testSuites { + for _, test := range suite.Tests { + if test.Status == junit.StatusFailed { + sigFail.Testcase = append(sigFail.Testcase, test) + } + } + } + sigFailures = append(sigFailures, sigFail) + } + + } + + reportTemplate, err := template.New("sigFailures").Parse(sigFailureReportTemplate) + if err != nil { + return fmt.Errorf("could not read template: %w", err) + } + + outputFile, err := os.Create(constructReportFilePath(opt)) + if err != nil { + return fmt.Errorf("could not create report file: %w", err) + } + defer outputFile.Close() + + err = reportTemplate.Execute(outputFile, sigFailures) + if err != nil { + return fmt.Errorf("could not execute template: %w", err) + } + + return nil +} diff --git a/pkg/htmlreport/sig-failure-report.gohtml b/pkg/htmlreport/sig-failure-report.gohtml new file mode 100644 index 00000000000..05403cd6370 --- /dev/null +++ b/pkg/htmlreport/sig-failure-report.gohtml @@ -0,0 +1,143 @@ +{{- /* + + This file is part of the KubeVirt project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Copyright The KubeVirt Authors. + +*/ -}} + +{{- /* sig-failure-report.gohtml */ -}} + + + + + SIG Failure Report + + + +

SIG Failure Report

+ {{- if . }} + {{- range . }} +
+
+ {{ .Sig }} + {{ .JobName }} +
+ {{ .FailureURL }} + {{ if .Testcase }} + + + + + + + + + {{- range .Testcase }} + + + + + {{- end }} + +
Test NameFailure Message
{{ .Name }} + {{- if .Error }} + {{ .Error }} + {{- else }} + Passed + {{- end }} +
+ {{ else }} +

No failed test cases found for this job.

+ {{ end }} +
+ {{- end }} + {{- else }} +

No SIG failures to display.

+ {{- end }} + + diff --git a/pkg/sigretests/main.go b/pkg/sigretests/main.go index e55a8b81025..56465b1caa1 100644 --- a/pkg/sigretests/main.go +++ b/pkg/sigretests/main.go @@ -159,7 +159,7 @@ func filterForLastCommit(org string, repo string, prNumber string, latestCommit func getJobsForLatestCommit(org string, repo string, prNumber string) (jobsLatestCommit []job, err error) { prHistory := prHistoryURL(org, repo, prNumber) - resp, err := httpGetWithRetry(prHistory) + resp, err := HttpGetWithRetry(prHistory) if err != nil { return nil, err } @@ -183,7 +183,7 @@ func getJobsForLatestCommit(org string, repo string, prNumber string) (jobsLates return jobsLatestCommit, nil } -func httpGetWithRetry(url string) (resp *http.Response, err error) { +func HttpGetWithRetry(url string) (resp *http.Response, err error) { httpRetryLog := log.WithField("url", url) retry.Do( func() error { diff --git a/pkg/types/main.go b/pkg/types/main.go index b923ac86a45..69dc2fa5770 100644 --- a/pkg/types/main.go +++ b/pkg/types/main.go @@ -98,6 +98,10 @@ type Options struct { Mode string TargetMetric string StartDate string + + // html-report options + Sig string + ResultsPath string } type Label struct { diff --git a/vendor/github.com/joshdk/go-junit/.gitignore b/vendor/github.com/joshdk/go-junit/.gitignore new file mode 100644 index 00000000000..b860b1620cb --- /dev/null +++ b/vendor/github.com/joshdk/go-junit/.gitignore @@ -0,0 +1,2 @@ +test-results/ +vendor/ diff --git a/vendor/github.com/joshdk/go-junit/.golangci.yml b/vendor/github.com/joshdk/go-junit/.golangci.yml new file mode 100644 index 00000000000..661b9bbd075 --- /dev/null +++ b/vendor/github.com/joshdk/go-junit/.golangci.yml @@ -0,0 +1,15 @@ +linters: + enable-all: true + disable: + - lll + - wsl + - gomnd + +issues: + exclude-use-default: true + exclude: + # Triggered by table tests calling t.Run. See + # https://github.com/kyoh86/scopelint/issues/4 for more information. + - Using the variable on range scope `test` in function literal + # Triggered by long table tests. + - Function 'Test\w+' is too long diff --git a/vendor/github.com/joshdk/go-junit/LICENSE.txt b/vendor/github.com/joshdk/go-junit/LICENSE.txt new file mode 100644 index 00000000000..642744f0152 --- /dev/null +++ b/vendor/github.com/joshdk/go-junit/LICENSE.txt @@ -0,0 +1,20 @@ +MIT License + +Copyright (c) Josh Komoroske + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/joshdk/go-junit/Makefile b/vendor/github.com/joshdk/go-junit/Makefile new file mode 100644 index 00000000000..4f32aab15a0 --- /dev/null +++ b/vendor/github.com/joshdk/go-junit/Makefile @@ -0,0 +1,61 @@ +#### Environment #### + +# Enforce usage of the Go modules system. +export GO111MODULE := on + +# Determine where `go get` will install binaries to. +GOBIN := $(HOME)/go/bin +ifdef GOPATH + GOBIN := $(GOPATH)/bin +endif + +# All target for when make is run on its own. +.PHONY: all +all: test lint + +#### Binary Dependencies #### + +# Install binary for go-junit-report. +go-junit-report := $(GOBIN)/go-junit-report +$(go-junit-report): + cd /tmp && go get -u github.com/jstemmer/go-junit-report + +# Install binary for golangci-lint. +golangci-lint := $(GOBIN)/golangci-lint +$(golangci-lint): + @./scripts/install-golangci-lint $(golangci-lint) + +# Install binary for goimports. +goimports := $(GOBIN)/goimports +$(goimports): + cd /tmp && go get -u golang.org/x/tools/cmd/goimports + +#### Linting #### + +# Run code linters. +.PHONY: lint +lint: $(golangci-lint) style + golangci-lint run + +# Run code formatters. Unformatted code will fail in CircleCI. +.PHONY: style +style: $(goimports) +ifdef CI + goimports -l . +else + goimports -l -w . +endif + +#### Testing #### + +# Run Go tests and generate a JUnit XML style test report for ingestion by CircleCI. +.PHONY: test +test: $(go-junit-report) + @mkdir -p test-results + @go test -race -v 2>&1 | tee test-results/report.log + @cat test-results/report.log | go-junit-report -set-exit-code > test-results/report.xml + +# Clean up test reports. +.PHONY: clean +clean: + @rm -rf test-results diff --git a/vendor/github.com/joshdk/go-junit/README.md b/vendor/github.com/joshdk/go-junit/README.md new file mode 100644 index 00000000000..b8805e0b6b1 --- /dev/null +++ b/vendor/github.com/joshdk/go-junit/README.md @@ -0,0 +1,180 @@ +[![License][license-badge]][license-link] +[![Godoc][godoc-badge]][godoc-link] +[![Go Report Card][go-report-badge]][go-report-link] +[![CircleCI][circleci-badge]][circleci-link] + +# Go JUnit + +🐜 Go library for ingesting JUnit XML reports + +## Installing + +You can fetch this library by running the following + +```bash +go get -u github.com/joshdk/go-junit +``` + +## Usage + +### Data Ingestion + +This library has a number of ingestion methods for convenience. + +The simplest of which parses raw JUnit XML data. + +```go +xml := []byte(` + + + + + + + + + + + Assertion failed + + + + + + + +`) + +suites, err := junit.Ingest(xml) +if err != nil { + log.Fatalf("failed to ingest JUnit xml %v", err) +} +``` + +You can then inspect the contents of the ingested suites. + +```go +for _, suite := range suites { + fmt.Println(suite.Name) + for _, test := range suite.Tests { + fmt.Printf(" %s\n", test.Name) + if test.Error != nil { + fmt.Printf(" %s: %s\n", test.Status, test.Error.Error()) + } else { + fmt.Printf(" %s\n", test.Status) + } + } +} +``` + +And observe some output like this. + +``` +JUnitXmlReporter +JUnitXmlReporter.constructor + should default path to an empty string + failed: Assertion failed + should default consolidate to true + skipped + should default useDotNotation to true + passed +``` + +### More Examples + +Additionally, you can ingest an entire file. + +```go +suites, err := junit.IngestFile("test-reports/report.xml") +if err != nil { + log.Fatalf("failed to ingest JUnit xml %v", err) +} +``` + +Or a list of multiple files. + +```go +suites, err := junit.IngestFiles([]string{ + "test-reports/report-1.xml", + "test-reports/report-2.xml", +}) +if err != nil { + log.Fatalf("failed to ingest JUnit xml %v", err) +} +``` + +Or any `.xml` files inside of a directory. + +```go +suites, err := junit.IngestDir("test-reports/") +if err != nil { + log.Fatalf("failed to ingest JUnit xml %v", err) +} +``` + +### Data Formats + +Due to the lack of implementation consistency in software that generates JUnit XML files, this library needs to take a somewhat looser approach to ingestion. As a consequence, many different possible JUnit formats can easily be ingested. + +A single top level `testsuite` tag, containing multiple `testcase` instances. + +```xml + + + + +``` + +A single top level `testsuites` tag, containing multiple `testsuite` instances. + +```xml + + + + + + +``` + +(Despite not technically being valid XML) Multiple top level `testsuite` tags, containing multiple `testcase` instances. + +```xml + + + + + + + + +``` + +In all cases, omitting (or even duplicated) the XML declaration tag is allowed. + +```xml + +``` + +## Contributing + +Found a bug or want to make go-junit better? Please [open a pull request](https://github.com/joshdk/go-junit/compare)! + +To make things easier, try out the following: + +- Running `make test` will run the test suite to verify behavior. + +- Running `make lint` will format the code, and report any linting issues using [golangci/golangci-lint](https://github.com/golangci/golangci-lint). + +## License + +This code is distributed under the [MIT License][license-link], see [LICENSE.txt][license-file] for more information. + +[circleci-badge]: https://circleci.com/gh/joshdk/go-junit.svg?&style=shield +[circleci-link]: https://circleci.com/gh/joshdk/go-junit/tree/master +[go-report-badge]: https://goreportcard.com/badge/github.com/joshdk/go-junit +[go-report-link]: https://goreportcard.com/report/github.com/joshdk/go-junit +[godoc-badge]: https://godoc.org/github.com/joshdk/go-junit?status.svg +[godoc-link]: https://godoc.org/github.com/joshdk/go-junit +[license-badge]: https://img.shields.io/badge/license-MIT-green.svg +[license-file]: https://github.com/joshdk/go-junit/blob/master/LICENSE.txt +[license-link]: https://opensource.org/licenses/MIT diff --git a/vendor/github.com/joshdk/go-junit/ingest.go b/vendor/github.com/joshdk/go-junit/ingest.go new file mode 100644 index 00000000000..0d21a522823 --- /dev/null +++ b/vendor/github.com/joshdk/go-junit/ingest.go @@ -0,0 +1,124 @@ +// Copyright Josh Komoroske. All rights reserved. +// Use of this source code is governed by the MIT license, +// a copy of which can be found in the LICENSE.txt file. + +package junit + +import ( + "strconv" + "strings" + "time" +) + +// findSuites performs a depth-first search through the XML document, and +// attempts to ingest any "testsuite" tags that are encountered. +func findSuites(nodes []xmlNode, suites chan Suite) { + for _, node := range nodes { + switch node.XMLName.Local { + case "testsuite": + suites <- ingestSuite(node) + default: + findSuites(node.Nodes, suites) + } + } +} + +func ingestSuite(root xmlNode) Suite { + suite := Suite{ + Name: root.Attr("name"), + Package: root.Attr("package"), + Properties: root.Attrs, + } + + for _, node := range root.Nodes { + switch node.XMLName.Local { + case "testsuite": + testsuite := ingestSuite(node) + suite.Suites = append(suite.Suites, testsuite) + case "testcase": + testcase := ingestTestcase(node) + suite.Tests = append(suite.Tests, testcase) + case "properties": + props := ingestProperties(node) + suite.Properties = props + case "system-out": + suite.SystemOut = string(node.Content) + case "system-err": + suite.SystemErr = string(node.Content) + } + } + + suite.Aggregate() + return suite +} + +func ingestProperties(root xmlNode) map[string]string { + props := make(map[string]string, len(root.Nodes)) + + for _, node := range root.Nodes { + if node.XMLName.Local == "property" { + name := node.Attr("name") + value := node.Attr("value") + props[name] = value + } + } + + return props +} + +func ingestTestcase(root xmlNode) Test { + test := Test{ + Name: root.Attr("name"), + Classname: root.Attr("classname"), + Duration: duration(root.Attr("time")), + Status: StatusPassed, + Properties: root.Attrs, + } + + for _, node := range root.Nodes { + switch node.XMLName.Local { + case "skipped": + test.Status = StatusSkipped + test.Message = node.Attr("message") + case "failure": + test.Status = StatusFailed + test.Message = node.Attr("message") + test.Error = ingestError(node) + case "error": + test.Status = StatusError + test.Message = node.Attr("message") + test.Error = ingestError(node) + case "system-out": + test.SystemOut = string(node.Content) + case "system-err": + test.SystemErr = string(node.Content) + } + } + + return test +} + +func ingestError(root xmlNode) Error { + return Error{ + Body: string(root.Content), + Type: root.Attr("type"), + Message: root.Attr("message"), + } +} + +func duration(t string) time.Duration { + // Remove commas for larger durations + t = strings.ReplaceAll(t, ",", "") + + // Check if there was a valid decimal value + if s, err := strconv.ParseFloat(t, 64); err == nil { + return time.Duration(s*1000000) * time.Microsecond + } + + // Check if there was a valid duration string + if d, err := time.ParseDuration(t); err == nil { + return d + } + + return 0 +} diff --git a/vendor/github.com/joshdk/go-junit/ingesters.go b/vendor/github.com/joshdk/go-junit/ingesters.go new file mode 100644 index 00000000000..9a50185cfbb --- /dev/null +++ b/vendor/github.com/joshdk/go-junit/ingesters.go @@ -0,0 +1,97 @@ +// Copyright Josh Komoroske. All rights reserved. +// Use of this source code is governed by the MIT license, +// a copy of which can be found in the LICENSE.txt file. + +package junit + +import ( + "bytes" + "io" + "os" + "path/filepath" + "strings" +) + +// IngestDir will search the given directory for XML files and return a slice +// of all contained JUnit test suite definitions. +func IngestDir(directory string) ([]Suite, error) { + var filenames []string + + err := filepath.Walk(directory, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // Add all regular files that end with ".xml" + if info.Mode().IsRegular() && strings.HasSuffix(info.Name(), ".xml") { + filenames = append(filenames, path) + } + + return nil + }) + + if err != nil { + return nil, err + } + + return IngestFiles(filenames) +} + +// IngestFiles will parse the given XML files and return a slice of all +// contained JUnit test suite definitions. +func IngestFiles(filenames []string) ([]Suite, error) { + var all = make([]Suite, 0) + + for _, filename := range filenames { + suites, err := IngestFile(filename) + if err != nil { + return nil, err + } + all = append(all, suites...) + } + + return all, nil +} + +// IngestFile will parse the given XML file and return a slice of all contained +// JUnit test suite definitions. +func IngestFile(filename string) ([]Suite, error) { + file, err := os.Open(filename) + if err != nil { + return nil, err + } + defer file.Close() + + return IngestReader(file) +} + +// IngestReader will parse the given XML reader and return a slice of all +// contained JUnit test suite definitions. +func IngestReader(reader io.Reader) ([]Suite, error) { + var ( + suiteChan = make(chan Suite) + suites = make([]Suite, 0) + ) + + nodes, err := parse(reader) + if err != nil { + return nil, err + } + + go func() { + findSuites(nodes, suiteChan) + close(suiteChan) + }() + + for suite := range suiteChan { + suites = append(suites, suite) + } + + return suites, nil +} + +// Ingest will parse the given XML data and return a slice of all contained +// JUnit test suite definitions. +func Ingest(data []byte) ([]Suite, error) { + return IngestReader(bytes.NewReader(data)) +} diff --git a/vendor/github.com/joshdk/go-junit/node.go b/vendor/github.com/joshdk/go-junit/node.go new file mode 100644 index 00000000000..6041d822640 --- /dev/null +++ b/vendor/github.com/joshdk/go-junit/node.go @@ -0,0 +1,47 @@ +// Copyright Josh Komoroske. All rights reserved. +// Use of this source code is governed by the MIT license, +// a copy of which can be found in the LICENSE.txt file. + +package junit + +import "encoding/xml" + +type xmlNode struct { + XMLName xml.Name + Attrs map[string]string `xml:"-"` + Content []byte `xml:",innerxml"` + Nodes []xmlNode `xml:",any"` +} + +func (n *xmlNode) Attr(name string) string { + return n.Attrs[name] +} + +func (n *xmlNode) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + type nodeAlias xmlNode + if err := d.DecodeElement((*nodeAlias)(n), &start); err != nil { + return err + } + + content, err := extractContent(n.Content) + if err != nil { + return err + } + + n.Content = content + + n.Attrs = attrMap(start.Attr) + return nil +} + +func attrMap(attrs []xml.Attr) map[string]string { + if len(attrs) == 0 { + return nil + } + + attributes := make(map[string]string, len(attrs)) + for _, attr := range attrs { + attributes[attr.Name.Local] = attr.Value + } + return attributes +} diff --git a/vendor/github.com/joshdk/go-junit/parse.go b/vendor/github.com/joshdk/go-junit/parse.go new file mode 100644 index 00000000000..e65937850a2 --- /dev/null +++ b/vendor/github.com/joshdk/go-junit/parse.go @@ -0,0 +1,115 @@ +// Copyright Josh Komoroske. All rights reserved. +// Use of this source code is governed by the MIT license, +// a copy of which can be found in the LICENSE.txt file. + +package junit + +import ( + "bytes" + "encoding/xml" + "errors" + "html" + "io" +) + +// reparentXML will wrap the given reader (which is assumed to be valid XML), +// in a fake root nodeAlias. +// +// This action is useful in the event that the original XML document does not +// have a single root nodeAlias, which is required by the XML specification. +// Additionally, Go's XML parser will silently drop all nodes after the first +// that is encountered, which can lead to data loss from a parser perspective. +// This function also enables the ingestion of blank XML files, which would +// normally cause a parsing error. +func reparentXML(reader io.Reader) io.Reader { + return io.MultiReader( + bytes.NewReader([]byte("")), + reader, + bytes.NewReader([]byte("")), + ) +} + +// extractContent parses the raw contents from an XML node, and returns it in a +// more consumable form. +// +// This function deals with two distinct classes of node data; Encoded entities +// and CDATA tags. These Encoded entities are normal (html escaped) text that +// you typically find between tags like so: +// • "Hello, world!" → "Hello, world!" +// • "I </3 XML" → "I " → "Hello, world!" +// • "" → "I </3 XML" +// • "" → "I . You probably too." → "I ") + mode int + output []byte + ) + + for { + if mode == 0 { + offset := bytes.Index(data, cdataStart) + if offset == -1 { + // The string "" appears in the data. This is an error! + return nil, errors.New("unmatched CDATA end tag") + } + + output = append(output, html.UnescapeString(string(data))...) + break + } + + // The string "" does not appear in the data. This is an error! + return nil, errors.New("unmatched CDATA start tag") + } + + // The string "]]>" appears at some offset. Read up to that offset. Discard "]]>" prefix. + output = append(output, data[:offset]...) + data = data[offset:] + data = data[3:] + mode = 0 + } + } + + return output, nil +} + +// parse unmarshalls the given XML data into a graph of nodes, and then returns +// a slice of all top-level nodes. +func parse(reader io.Reader) ([]xmlNode, error) { + var ( + dec = xml.NewDecoder(reparentXML(reader)) + root xmlNode + ) + + if err := dec.Decode(&root); err != nil { + return nil, err + } + + return root.Nodes, nil +} diff --git a/vendor/github.com/joshdk/go-junit/types.go b/vendor/github.com/joshdk/go-junit/types.go new file mode 100644 index 00000000000..35d1299f2a6 --- /dev/null +++ b/vendor/github.com/joshdk/go-junit/types.go @@ -0,0 +1,190 @@ +// Copyright Josh Komoroske. All rights reserved. +// Use of this source code is governed by the MIT license, +// a copy of which can be found in the LICENSE.txt file. + +package junit + +import ( + "strings" + "time" +) + +// Status represents the result of a single a JUnit testcase. Indicates if a +// testcase was run, and if it was successful. +type Status string + +const ( + // StatusPassed represents a JUnit testcase that was run, and did not + // result in an error or a failure. + StatusPassed Status = "passed" + + // StatusSkipped represents a JUnit testcase that was intentionally + // skipped. + StatusSkipped Status = "skipped" + + // StatusFailed represents a JUnit testcase that was run, but resulted in + // a failure. Failures are violations of declared test expectations, + // such as a failed assertion. + StatusFailed Status = "failed" + + // StatusError represents a JUnit testcase that was run, but resulted in + // an error. Errors are unexpected violations of the test itself, such as + // an uncaught exception. + StatusError Status = "error" +) + +// Totals contains aggregated results across a set of test runs. Is usually +// calculated as a sum of all given test runs, and overrides whatever was given +// at the suite level. +// +// The following relation should hold true. +// Tests == (Passed + Skipped + Failed + Error) +type Totals struct { + // Tests is the total number of tests run. + Tests int `json:"tests" yaml:"tests"` + + // Passed is the total number of tests that passed successfully. + Passed int `json:"passed" yaml:"passed"` + + // Skipped is the total number of tests that were skipped. + Skipped int `json:"skipped" yaml:"skipped"` + + // Failed is the total number of tests that resulted in a failure. + Failed int `json:"failed" yaml:"failed"` + + // Error is the total number of tests that resulted in an error. + Error int `json:"error" yaml:"error"` + + // Duration is the total time taken to run all tests. + Duration time.Duration `json:"duration" yaml:"duration"` +} + +// Suite represents a logical grouping (suite) of tests. +type Suite struct { + // Name is a descriptor given to the suite. + Name string `json:"name" yaml:"name"` + + // Package is an additional descriptor for the hierarchy of the suite. + Package string `json:"package" yaml:"package"` + + // Properties is a mapping of key-value pairs that were available when the + // tests were run. + Properties map[string]string `json:"properties,omitempty" yaml:"properties,omitempty"` + + // Tests is an ordered collection of tests with associated results. + Tests []Test `json:"tests,omitempty" yaml:"tests,omitempty"` + + // Suites is an ordered collection of suites with associated tests. + Suites []Suite `json:"suites,omitempty" yaml:"suites,omitempty"` + + // SystemOut is textual test output for the suite. Usually output that is + // written to stdout. + SystemOut string `json:"stdout,omitempty" yaml:"stdout,omitempty"` + + // SystemErr is textual test error output for the suite. Usually output that is + // written to stderr. + SystemErr string `json:"stderr,omitempty" yaml:"stderr,omitempty"` + + // Totals is the aggregated results of all tests. + Totals Totals `json:"totals" yaml:"totals"` +} + +// Aggregate calculates result sums across all tests and nested suites. +func (s *Suite) Aggregate() { + totals := Totals{Tests: len(s.Tests)} + + for _, test := range s.Tests { + totals.Duration += test.Duration + switch test.Status { + case StatusPassed: + totals.Passed++ + case StatusSkipped: + totals.Skipped++ + case StatusFailed: + totals.Failed++ + case StatusError: + totals.Error++ + } + } + + // just summing totals from nested suites + for _, suite := range s.Suites { + suite.Aggregate() + totals.Tests += suite.Totals.Tests + totals.Duration += suite.Totals.Duration + totals.Passed += suite.Totals.Passed + totals.Skipped += suite.Totals.Skipped + totals.Failed += suite.Totals.Failed + totals.Error += suite.Totals.Error + } + + s.Totals = totals +} + +// Test represents the results of a single test run. +type Test struct { + // Name is a descriptor given to the test. + Name string `json:"name" yaml:"name"` + + // Classname is an additional descriptor for the hierarchy of the test. + Classname string `json:"classname" yaml:"classname"` + + // Duration is the total time taken to run the tests. + Duration time.Duration `json:"duration" yaml:"duration"` + + // Status is the result of the test. Status values are passed, skipped, + // failure, & error. + Status Status `json:"status" yaml:"status"` + + // Message is an textual description optionally included with a skipped, + // failure, or error test case. + Message string `json:"message" yaml:"message"` + + // Error is a record of the failure or error of a test, if applicable. + // + // The following relations should hold true. + // Error == nil && (Status == Passed || Status == Skipped) + // Error != nil && (Status == Failed || Status == Error) + Error error `json:"error" yaml:"error"` + + // Additional properties from XML node attributes. + // Some tools use them to store additional information about test location. + Properties map[string]string `json:"properties" yaml:"properties"` + + // SystemOut is textual output for the test case. Usually output that is + // written to stdout. + SystemOut string `json:"stdout,omitempty" yaml:"stdout,omitempty"` + + // SystemErr is textual error output for the test case. Usually output that is + // written to stderr. + SystemErr string `json:"stderr,omitempty" yaml:"stderr,omitempty"` +} + +// Error represents an erroneous test result. +type Error struct { + // Message is a descriptor given to the error. Purpose and values differ by + // environment. + Message string `json:"message,omitempty" yaml:"message,omitempty"` + + // Type is a descriptor given to the error. Purpose and values differ by + // framework. Value is typically an exception class, such as an assertion. + Type string `json:"type,omitempty" yaml:"type,omitempty"` + + // Body is extended text for the error. Purpose and values differ by + // framework. Value is typically a stacktrace. + Body string `json:"body,omitempty" yaml:"body,omitempty"` +} + +// Error returns a textual description of the test error. +func (err Error) Error() string { + switch { + case strings.TrimSpace(err.Body) != "": + return err.Body + + case strings.TrimSpace(err.Message) != "": + return err.Message + + default: + return err.Type + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 956855dc3eb..1e2f8f4d03e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -65,6 +65,9 @@ github.com/google/go-cmp/cmp/internal/value # github.com/inconshreveable/mousetrap v1.1.0 ## explicit; go 1.18 github.com/inconshreveable/mousetrap +# github.com/joshdk/go-junit v1.0.0 +## explicit; go 1.12 +github.com/joshdk/go-junit # github.com/kr/text v0.2.0 ## explicit # github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822