htmlreport: Add new command that creates HTML reports of SIG test case failures

This new command will primarily used by periodic prow jobs to generate
daily reports on each specific SIG's tescase failures. These reports
will help SIG members to identify and tackle the faiures.

The reports will be stored on the kubevirt GCS bucket and can be linked
to from the ci-health README where we track the number of SIG e2e
failures

Signed-off-by: Brian Carey <bcarey@redhat.com>

Author: brianmcarey

README.md

@@ -165,3 +165,12 @@ To generate plots you should execute:
 $ go run ./cmd/batch --gh-token /path/to/token --source <org/repo> --path $(pwd)/output --mode plot --target-metric merged-prs --start-date 2020-05-19
 ```
 Plot mode requires data previously generated by fetch mode.
+
+### html-report command
+
+A command that generates HTML reports of test case failures per SIG. To create a report for SIG Compute failures:
+```
+go run ./cmd/html-report --sig compute --results-path ./output/kubevirt/kubevirt/results.json --path /tmp/
+```
+
+This should create a HTML report called sig-compute-failure-report.html under /tmp/.

cmd/html-report/main.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"fmt"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	"github.com/kubevirt/ci-health/pkg/constants"
+	"github.com/kubevirt/ci-health/pkg/htmlreport"
+	"github.com/kubevirt/ci-health/pkg/types"
+)
+
+func main() {
+	opt := &types.Options{
+		Path:        constants.DefaultPath,
+		Sig:         "compute",
+		ResultsPath: constants.DefaultResultsPath,
+	}
+
+	cmd := &cobra.Command{
+		Run: func(cmd *cobra.Command, arguments []string) {
+			if err := htmlreport.Generate(opt); err != nil {
+				log.Fatalf("error generating HTML report: %v", err)
+			}
+			reportPath := fmt.Sprintf("%s/sig-%s-failure-report.html", opt.Path, opt.Sig)
+			log.Printf("HTML report written to %s", reportPath)
+		},
+	}
+	flag := cmd.Flags()
+
+	flag.StringVar(&opt.Path, "path", opt.Path, "Path to output HTML reports to")
+	flag.StringVar(&opt.Sig, "sig", opt.Sig, "Name of SIG to generate failure report for")
+	flag.StringVar(&opt.ResultsPath, "results-path", opt.ResultsPath, "Path to the results.json file created by ci-health")
+
+	if err := cmd.Execute(); err != nil {
+		log.Fatalf("error: %v", err)
+	}
+}

go.mod

@@ -6,6 +6,7 @@ toolchain go1.24.3
 
 require (
 	github.com/avast/retry-go v3.0.0+incompatible
+	github.com/joshdk/go-junit v1.0.0
 	github.com/narqo/go-badge v0.0.0-20230821190521-c9a75c019a59
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.27.10

go.sum

@@ -64,6 +64,8 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE=
+github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
 github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=

pkg/constants/main.go

@@ -18,6 +18,7 @@ const (
 	SIGRetests           = "SIGRetests"
 
 	DefaultPath                         = "/tmp/test"
+	DefaultResultsPath                  = ""
 	DefaultTokenPath                    = ""
 	DefaultSource                       = "kubevirt/kubevirt"
 	DefaultDataDays                     = 7

pkg/htmlreport/main.go

@@ -0,0 +1,182 @@
+package htmlreport
+
+import (
+	_ "embed"
+	"encoding/json"
+	"encoding/xml"
+	"fmt"
+	"html/template"
+	"io"
+	"net/http"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/joshdk/go-junit"
+	"github.com/kubevirt/ci-health/pkg/sigretests"
+	"github.com/kubevirt/ci-health/pkg/types"
+	log "github.com/sirupsen/logrus"
+)
+
+var (
+	//go:embed sig-failure-report.gohtml
+	sigFailureReportTemplate string
+)
+
+type HTMLReportResults struct {
+	Data struct {
+		SIGRetests struct {
+			FailedJobLeaderBoard []types.FailedJob `json:"FailedJobLeaderBoard"`
+		} `json:"SIGRetests"`
+	} `json:"Data"`
+}
+
+type Failure struct {
+	XMLName xml.Name `xml:"failure"`
+	Message string   `xml:"message,attr"`
+	Type    string   `xml:"type,attr"`
+	Value   string   `xml:",chardata"`
+}
+
+type SigFailure struct {
+	Sig        string
+	JobName    string
+	FailureURL string
+	Testcase   []junit.Test
+}
+
+var jobRegexAliases = map[string]string{
+	"compute": "sig-compute$|sig-compute-serial$|sig-compute-migrations$|sig-operator$|vgpu$|sev$",
+	"network": "sig-network$|sriov$",
+	"storage": "sig-storage$",
+}
+
+func fetchResults(resultsPath string) (*HTMLReportResults, error) {
+	body, err := os.ReadFile(resultsPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read results.json file: %w", err)
+	}
+
+	var results HTMLReportResults
+	if err := json.Unmarshal(body, &results); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal results.json: %w", err)
+	}
+
+	return &results, nil
+}
+
+func fetchJunit(url string) ([]junit.Suite, error) {
+	resp, err := sigretests.HttpGetWithRetry(url)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch %s: %s", url, err)
+	}
+	defer resp.Body.Close()
+
+	// Ignore missing junit files as it suggests an issue with the job
+	if resp.StatusCode == http.StatusNotFound {
+		return nil, nil
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("failed to fetch %s: status code %d", url, resp.StatusCode)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read %s body: %w", url, err)
+	}
+
+	testsuite, err := junit.Ingest(body)
+	if err == nil {
+		return testsuite, nil
+	}
+
+	return nil, fmt.Errorf("failed to unmarshal junit.functest.xml as <testsuites> or <testsuite>")
+}
+
+func constructReportFilePath(opt *types.Options) string {
+	return fmt.Sprintf("%s/sig-%s-failure-report.html", opt.Path, opt.Sig)
+}
+
+func constructJunitURL(failureURL string) string {
+	junitURL := strings.Replace(failureURL, "prow.ci.kubevirt.io//view/gs", "gcsweb.ci.kubevirt.io/gcs", 1)
+	if !strings.HasSuffix(junitURL, "/") {
+		junitURL += "/"
+	}
+	junitURL += "artifacts/junit.functest.xml"
+	return junitURL
+}
+
+func Generate(opt *types.Options) error {
+
+	if opt.ResultsPath == "" {
+		return fmt.Errorf("the path to results.json is required")
+	}
+
+	jobRegex, ok := jobRegexAliases[opt.Sig]
+	if !ok {
+		return fmt.Errorf("unknown SIG: %s", opt.Sig)
+	}
+
+	results, err := fetchResults(opt.ResultsPath)
+	if err != nil {
+		return fmt.Errorf("failed to parse results.json: %w", err)
+	}
+
+	compiledRegex, err := regexp.Compile(jobRegex)
+	if err != nil {
+		return fmt.Errorf("invalid job regex provided: %w", err)
+	}
+
+	var sigFailures []SigFailure
+
+	for _, job := range results.Data.SIGRetests.FailedJobLeaderBoard {
+		if !compiledRegex.MatchString(job.JobName) {
+			continue
+		}
+		for _, failureURL := range job.FailureURLs {
+			var sigFail SigFailure
+			junitURL := constructJunitURL(failureURL)
+			testSuites, err := fetchJunit(junitURL)
+			if err != nil {
+				log.Warnf("failed to fetch junit results: %s", err)
+				continue
+			}
+			if testSuites == nil {
+				// SIG CI failure
+				continue
+			}
+			sigFail.Sig = opt.Sig
+			sigFail.JobName = job.JobName
+			sigFail.FailureURL = failureURL
+
+			for _, suite := range testSuites {
+				for _, test := range suite.Tests {
+					if test.Status == junit.StatusFailed {
+						sigFail.Testcase = append(sigFail.Testcase, test)
+					}
+				}
+			}
+			sigFailures = append(sigFailures, sigFail)
+		}
+
+	}
+
+	reportTemplate, err := template.New("sigFailures").Parse(sigFailureReportTemplate)
+	if err != nil {
+		return fmt.Errorf("could not read template: %w", err)
+	}
+
+	outputFile, err := os.Create(constructReportFilePath(opt))
+	if err != nil {
+		return fmt.Errorf("could not create report file: %w", err)
+	}
+	defer outputFile.Close()
+
+	err = reportTemplate.Execute(outputFile, sigFailures)
+	if err != nil {
+		return fmt.Errorf("could not execute template: %w", err)
+	}
+
+	return nil
+}