htmlreport: Add new command that creates HTML reports of SIG test case failures

README.md

@@ -165,3 +165,12 @@ To generate plots you should execute:
 $ go run ./cmd/batch --gh-token /path/to/token --source <org/repo> --path $(pwd)/output --mode plot --target-metric merged-prs --start-date 2020-05-19
 ```
 Plot mode requires data previously generated by fetch mode.
+
+### html-report command
+
+A command that generates HTML reports of test case failures per SIG. To create a report for SIG Compute failures:
+```
+go run ./cmd/html-report --sig compute --results-path ./output/kubevirt/kubevirt/results.json --path /tmp/
+```
+
+This should create a HTML report called sig-compute-failure-report.html under /tmp/.

cmd/html-report/main.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"fmt"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	"github.com/kubevirt/ci-health/pkg/constants"
+	"github.com/kubevirt/ci-health/pkg/htmlreport"
+	"github.com/kubevirt/ci-health/pkg/types"
+)
+
+func main() {
+	opt := &types.Options{
+		Path:        constants.DefaultPath,
+		Sig:         "compute",
+		ResultsPath: constants.DefaultResultsPath,
+	}
+
+	cmd := &cobra.Command{
+		Run: func(cmd *cobra.Command, arguments []string) {
+			if err := htmlreport.Generate(opt); err != nil {
+				log.Fatalf("error generating HTML report: %v", err)
+			}
+			reportPath := fmt.Sprintf("%s/sig-%s-failure-report.html", opt.Path, opt.Sig)
+			log.Printf("HTML report written to %s", reportPath)
+		},
+	}
+	flag := cmd.Flags()
+
+	flag.StringVar(&opt.Path, "path", opt.Path, "Path to output HTML reports to")
+	flag.StringVar(&opt.Sig, "sig", opt.Sig, "Name of SIG to generate failure report for")
+	flag.StringVar(&opt.ResultsPath, "results-path", opt.ResultsPath, "Path to the results.json file created by ci-health")
+
+	if err := cmd.Execute(); err != nil {
+		log.Fatalf("error: %v", err)
+	}
+}

go.mod

@@ -6,6 +6,7 @@ toolchain go1.24.3
 
 require (
 	github.com/avast/retry-go v3.0.0+incompatible
+	github.com/joshdk/go-junit v1.0.0
 	github.com/narqo/go-badge v0.0.0-20230821190521-c9a75c019a59
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.27.10

go.sum

@@ -64,6 +64,8 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE=
+github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
 github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=

pkg/constants/main.go

@@ -18,6 +18,7 @@ const (
 	SIGRetests           = "SIGRetests"
 
 	DefaultPath                         = "/tmp/test"
+	DefaultResultsPath                  = ""
 	DefaultTokenPath                    = ""
 	DefaultSource                       = "kubevirt/kubevirt"
 	DefaultDataDays                     = 7

pkg/htmlreport/main.go

@@ -0,0 +1,182 @@
+package htmlreport
+
+import (
+	_ "embed"
+	"encoding/json"
+	"encoding/xml"
+	"fmt"
+	"html/template"
+	"io"
+	"net/http"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/joshdk/go-junit"
+	"github.com/kubevirt/ci-health/pkg/sigretests"
+	"github.com/kubevirt/ci-health/pkg/types"
+	log "github.com/sirupsen/logrus"
+)
+
+var (
+	//go:embed sig-failure-report.gohtml
+	sigFailureReportTemplate string
+)
+
+type HTMLReportResults struct {
+	Data struct {
+		SIGRetests struct {
+			FailedJobLeaderBoard []types.FailedJob `json:"FailedJobLeaderBoard"`
+		} `json:"SIGRetests"`
+	} `json:"Data"`
+}
+
+type Failure struct {
+	XMLName xml.Name `xml:"failure"`
+	Message string   `xml:"message,attr"`
+	Type    string   `xml:"type,attr"`
+	Value   string   `xml:",chardata"`
+}
+
+type SigFailure struct {
+	Sig        string
+	JobName    string
+	FailureURL string
+	Testcase   []junit.Test
+}
+
+var jobRegexAliases = map[string]string{
+	"compute": "sig-compute$|sig-compute-serial$|sig-compute-migrations$|sig-operator$|vgpu$|sev$",
+	"network": "sig-network$|sriov$",
+	"storage": "sig-storage$",
+}
+
+func fetchResults(resultsPath string) (*HTMLReportResults, error) {
+	body, err := os.ReadFile(resultsPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read results.json file: %w", err)
+	}
+
+	var results HTMLReportResults
+	if err := json.Unmarshal(body, &results); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal results.json: %w", err)
+	}
+
+	return &results, nil
+}
+
+func fetchJunit(url string) ([]junit.Suite, error) {
+	resp, err := sigretests.HttpGetWithRetry(url)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch %s: %s", url, err)
+	}
+	defer resp.Body.Close()
+
+	// Ignore missing junit files as it suggests an issue with the job
+	if resp.StatusCode == http.StatusNotFound {
+		return nil, nil
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("failed to fetch %s: status code %d", url, resp.StatusCode)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read %s body: %w", url, err)
+	}
+
+	testsuite, err := junit.Ingest(body)
+	if err == nil {
+		return testsuite, nil
+	}
+
+	return nil, fmt.Errorf("failed to unmarshal junit.functest.xml as <testsuites> or <testsuite>")
+}
+
+func constructReportFilePath(opt *types.Options) string {
+	return fmt.Sprintf("%s/sig-%s-failure-report.html", opt.Path, opt.Sig)
+}
+
+func constructJunitURL(failureURL string) string {
+	junitURL := strings.Replace(failureURL, "prow.ci.kubevirt.io//view/gs", "gcsweb.ci.kubevirt.io/gcs", 1)
+	if !strings.HasSuffix(junitURL, "/") {
+		junitURL += "/"
+	}
+	junitURL += "artifacts/junit.functest.xml"
+	return junitURL
+}
+
+func Generate(opt *types.Options) error {
+
+	if opt.ResultsPath == "" {
+		return fmt.Errorf("the path to results.json is required")
+	}
+
+	jobRegex, ok := jobRegexAliases[opt.Sig]
+	if !ok {
+		return fmt.Errorf("unknown SIG: %s", opt.Sig)
+	}
+
+	results, err := fetchResults(opt.ResultsPath)
+	if err != nil {
+		return fmt.Errorf("failed to parse results.json: %w", err)
+	}
+
+	compiledRegex, err := regexp.Compile(jobRegex)
+	if err != nil {
+		return fmt.Errorf("invalid job regex provided: %w", err)
+	}
+
+	var sigFailures []SigFailure
+
+	for _, job := range results.Data.SIGRetests.FailedJobLeaderBoard {
+		if !compiledRegex.MatchString(job.JobName) {
+			continue
+		}
+		for _, failureURL := range job.FailureURLs {
+			var sigFail SigFailure
+			junitURL := constructJunitURL(failureURL)
+			testSuites, err := fetchJunit(junitURL)
+			if err != nil {
+				log.Warnf("failed to fetch junit results: %s", err)
+				continue
+			}
+			if testSuites == nil {
+				// SIG CI failure
+				continue
+			}
+			sigFail.Sig = opt.Sig
+			sigFail.JobName = job.JobName
+			sigFail.FailureURL = failureURL
+
+			for _, suite := range testSuites {
+				for _, test := range suite.Tests {
+					if test.Status == junit.StatusFailed {
+						sigFail.Testcase = append(sigFail.Testcase, test)
+					}
+				}
+			}
+			sigFailures = append(sigFailures, sigFail)
+		}
+
+	}
+
+	reportTemplate, err := template.New("sigFailures").Parse(sigFailureReportTemplate)
+	if err != nil {
+		return fmt.Errorf("could not read template: %w", err)
+	}
+
+	outputFile, err := os.Create(constructReportFilePath(opt))
+	if err != nil {
+		return fmt.Errorf("could not create report file: %w", err)
+	}
+	defer outputFile.Close()
+
+	err = reportTemplate.Execute(outputFile, sigFailures)
+	if err != nil {
+		return fmt.Errorf("could not execute template: %w", err)
+	}
+
+	return nil
+}