feat(tests): improving E2E HTML report and output on remediation tests

.github/scripts/report/go.sum

@@ -1,28 +1,28 @@
-github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U=
-github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
-github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2/go.mod h1:0KeJpeMD6o+O4hW7qJOT7vyQPKrWmj26uf5wMc/IiIs=
-github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
-github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
-github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
-github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
-github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A=
-github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
-github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/tdewolff/minify/v2 v2.9.21 h1:nO4s1PEMy7aRjlIlbr3Jgr+bJby8QYuifa2Vs2f9lh4=
-github.com/tdewolff/minify/v2 v2.9.21/go.mod h1:PoDBts2L7sCwUT28vTAlozGeD6qxjrrihtin4bR/RMM=
-github.com/tdewolff/parse/v2 v2.5.19 h1:Kjaj3KQOx/4elIxlBSglus4E2oMfdROphvbq2b+OBZ0=
-github.com/tdewolff/parse/v2 v2.5.19/go.mod h1:WzaJpRSbwq++EIQHYIRTpbYKNA3gn9it1Ik++q4zyho=
-github.com/tdewolff/test v1.0.6 h1:76mzYJQ83Op284kMT+63iCNCI7NEERsIN8dLM+RiKr4=
-github.com/tdewolff/test v1.0.6/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
-golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
-golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U=
+github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2/go.mod h1:0KeJpeMD6o+O4hW7qJOT7vyQPKrWmj26uf5wMc/IiIs=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A=
+github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/tdewolff/minify/v2 v2.9.21 h1:nO4s1PEMy7aRjlIlbr3Jgr+bJby8QYuifa2Vs2f9lh4=
+github.com/tdewolff/minify/v2 v2.9.21/go.mod h1:PoDBts2L7sCwUT28vTAlozGeD6qxjrrihtin4bR/RMM=
+github.com/tdewolff/parse/v2 v2.5.19 h1:Kjaj3KQOx/4elIxlBSglus4E2oMfdROphvbq2b+OBZ0=
+github.com/tdewolff/parse/v2 v2.5.19/go.mod h1:WzaJpRSbwq++EIQHYIRTpbYKNA3gn9it1Ik++q4zyho=
+github.com/tdewolff/test v1.0.6 h1:76mzYJQ83Op284kMT+63iCNCI7NEERsIN8dLM+RiKr4=
+github.com/tdewolff/test v1.0.6/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

.github/scripts/report/main.go

@@ -7,6 +7,7 @@ import (
 	"math"
 	"os"
 	"path/filepath"
+	"regexp"
 	"sort"
 	"strings"
 )
@@ -23,8 +24,26 @@ type Counters struct {
 	CountTotal int
 }
 
+type ExpectedActual struct {
+	ExtraElements         ActualExpectedWithStatus // list of extra elements
+	TestInfo              []string
+	Messages 			  ActualExpectedWithStatus
+	FailOutput            []string
+}
+
+type ActualExpectedWithStatus struct {
+	ExpectedContent []CodeLineStatus
+	ActualContent   []CodeLineStatus
+}
+
+type CodeLineStatus struct {
+	Line   string
+	Status bool // differed or not
+}
+
 type TestsData struct {
 	TestLog TestLog
+	ExpectedActual ExpectedActual
 	FailLog []string
 }
 
@@ -51,6 +70,252 @@ func FindTest(tests []TestsData, testName string) (*TestsData, bool) {
 	return nil, false
 }
 
+func cleanOutput(s string) string {
+	// remove (len=N)
+	lenPattern := regexp.MustCompile(`\(len=\d+\)\s*`)
+	s = lenPattern.ReplaceAllString(s, "")
+	// Remove type annotations like (string), (int), (*string), (model.IssueType)...
+	typePattern := regexp.MustCompile(`\([a-zA-Z*\[][^)]*\)\s*`)
+	s = typePattern.ReplaceAllString(s, "")
+	return s
+}
+
+func extractPayloadDiffLines(failLog []string) ExpectedActual {
+	var testInfo []string
+	var messages ActualExpectedWithStatus
+	var failOutput []string
+
+	const (
+		stateNone = iota
+		stateMessagesExpected
+		stateMessagesActual
+		stateTestInfo
+		stateFailLog
+	)
+	state := stateTestInfo
+
+	for _, line := range failLog {
+		trimmed := strings.TrimSpace(line)
+		if strings.HasPrefix(trimmed, "actualPayload") {
+			state = stateMessagesActual
+		} else if strings.HasPrefix(trimmed, "expectedPayload") {
+			state = stateMessagesExpected
+		} else if strings.HasPrefix(trimmed, "--- FAIL:") {
+			state = stateFailLog
+		} else if strings.HasPrefix(trimmed, `"queries": [`) {
+			state = stateNone
+		}
+
+		switch state {
+		case stateMessagesActual:
+			if !strings.HasPrefix(trimmed, "actualPayload") {
+				messages.ActualContent = append(messages.ActualContent, CodeLineStatus{
+					Line: line,
+					Status: false,
+				})
+			}
+		case stateMessagesExpected:
+			if !strings.HasPrefix(trimmed, "expectedPayload") {
+				messages.ExpectedContent = append(messages.ExpectedContent, CodeLineStatus{
+					Line: line,
+					Status: false,
+				})
+			}
+		case stateFailLog:
+			failOutput = append(failOutput, line)
+			state = stateNone
+		case stateTestInfo:
+			testInfo = append(testInfo, line)
+		}
+	}
+	return ExpectedActual{
+		TestInfo: testInfo,
+		Messages: messages,
+		FailOutput: failOutput,
+	}
+}
+
+func extractExpectedActualLines(failLog []string) ExpectedActual {
+	var extraElements ActualExpectedWithStatus
+	//var extraExpected []string // extra elements in list A
+	//var extraActual []string
+	var testInfo []string
+	var messages ActualExpectedWithStatus
+	var failOutput []string
+
+	const (
+		stateNone         = iota
+		stateExtraA
+		stateExtraB
+		stateTestInfo
+		stateMessagesExpected
+		stateMessagesActual
+		stateFailLog
+	)
+	state := stateNone
+
+	for _, line := range failLog {
+		trimmed := strings.TrimSpace(line)
+
+		switch trimmed {
+		case "extra elements in list A:":
+			state = stateExtraA
+			continue
+		case "extra elements in list B:":
+			state = stateExtraB
+			continue
+		}
+
+		if strings.HasPrefix(trimmed, "Test:") {
+			state = stateTestInfo
+		} else if strings.HasSuffix(trimmed, "Expected Queries content: 'fixtures/{") {
+			state = stateMessagesExpected
+		} else if strings.HasSuffix(trimmed, "doesn't match the Actual Queries content: 'output/{") {
+			state = stateMessagesActual
+		} else if strings.HasPrefix(trimmed, "--- FAIL:") {
+			state = stateFailLog
+		}
+
+		if trimmed == "" && (state == stateExtraA || state == stateExtraB) {
+			state = stateNone
+			continue
+		}
+
+		switch state {
+		case stateExtraA:
+			if !strings.HasPrefix(trimmed, "([]interface {})") && !strings.HasPrefix(trimmed, "(model.VulnerableFile) {") {
+				cleaned_extraElement_line := cleanOutput(line)
+				extraElements.ExpectedContent = append(extraElements.ExpectedContent, CodeLineStatus{
+					Line: cleaned_extraElement_line,
+					Status: false,
+				})
+			}
+		case stateExtraB:
+			if !strings.HasPrefix(trimmed, "([]interface {})") && !strings.HasPrefix(trimmed, "(model.VulnerableFile) {") {
+				cleaned_extraElement_line := cleanOutput(line)
+				extraElements.ActualContent = append(extraElements.ActualContent, CodeLineStatus{
+					Line: cleaned_extraElement_line,
+					Status: false,
+				})
+			}
+		case stateTestInfo:
+			testInfo = append(testInfo, line)
+		case stateMessagesActual:
+			if !strings.HasSuffix(trimmed, "doesn't match the Actual Queries content: 'output/{") {
+				messages.ActualContent = append(messages.ActualContent, CodeLineStatus{
+					Line: line,
+					Status: false,
+				})
+			}
+		case stateMessagesExpected:
+			if !strings.HasSuffix(trimmed, "Expected Queries content: 'fixtures/{") {
+				messages.ExpectedContent = append(messages.ExpectedContent, CodeLineStatus{
+					Line: line, 
+					Status: false,
+				})
+			}
+		case stateFailLog:
+			failOutput = append(failOutput, line)
+			state = stateNone
+		}
+	}
+
+	return ExpectedActual{
+		ExtraElements: extraElements,
+		TestInfo: testInfo,
+		Messages: messages,
+		FailOutput: failOutput,
+	}
+}
+
+func isDifferentNumberOfLines(failLog []string) bool {
+	var hasExpectedFileNumberLines, hasActualFileNumberLines bool
+	for _, failLogEntry := range failLog {
+		trimmedEntry := strings.TrimSpace(failLogEntry)
+		if trimmedEntry == "" {
+			continue
+		}
+		if strings.Contains(trimmedEntry, "Expected file number of lines:") {
+			hasExpectedFileNumberLines = true
+		}
+		if strings.Contains(failLogEntry, "Actual file number of lines:") {
+			hasActualFileNumberLines = true
+		}
+		if hasExpectedFileNumberLines && hasActualFileNumberLines {
+			return true
+		}
+	}
+	return false
+}
+
+func isExpectedVsActual(failLog []string) bool {
+	var hasExtraA, hasExtraB bool
+	for _, failLogEntry := range failLog {
+		trimmedEntry := strings.TrimSpace(failLogEntry)
+		if trimmedEntry == "extra elements in list A:" {
+			hasExtraA = true
+		}
+		if trimmedEntry == "extra elements in list B:" {
+			hasExtraB = true
+		}
+		if hasExtraA && hasExtraB {
+			return true
+		}
+	}
+	return false
+}
+
+func compareMessageContent(expectedActual *ExpectedActual) {
+	expectedLen := len(expectedActual.Messages.ExpectedContent)
+	actualLen := len(expectedActual.Messages.ActualContent)
+	actualLenExtraElements := len(expectedActual.ExtraElements.ExpectedContent)
+	expectedLenExtraElements := len(expectedActual.ExtraElements.ActualContent)
+
+	maxLen := expectedLen
+    if actualLen > maxLen {
+        maxLen = actualLen
+    }
+
+	maxLenExtraElements := expectedLenExtraElements
+	if actualLenExtraElements > maxLenExtraElements {
+		maxLenExtraElements = actualLenExtraElements
+	}
+
+	for i := range maxLen {
+		// if one of the sides does not have a line in this position, the one that exists 
+		// is tagged as the different one
+		if i >= expectedLen || i >= actualLen {
+			if i < expectedLen {
+				expectedActual.Messages.ExpectedContent[i].Status = true
+			}
+			if i < actualLen {
+				expectedActual.Messages.ActualContent[i].Status = true
+			}
+			continue
+		}
+		if expectedActual.Messages.ExpectedContent[i].Line != expectedActual.Messages.ActualContent[i].Line {
+			expectedActual.Messages.ExpectedContent[i].Status = true
+			expectedActual.Messages.ActualContent[i].Status = true
+		}
+	}
+
+	for j := range maxLenExtraElements {
+		if j >= expectedLenExtraElements || j >= actualLenExtraElements {
+			if j < expectedLenExtraElements {
+				expectedActual.ExtraElements.ExpectedContent[j].Status = true
+			}
+			if j > actualLenExtraElements {
+				expectedActual.ExtraElements.ActualContent[j].Status = true
+			}
+			continue
+		}
+		if expectedActual.ExtraElements.ExpectedContent[j].Line != expectedActual.ExtraElements.ActualContent[j].Line {
+			expectedActual.ExtraElements.ExpectedContent[j].Status = true
+			expectedActual.ExtraElements.ActualContent[j].Status = true
+		}
+	}
+}
+
 func main() {
 	var testPath, testName, reportPath, reportName string
 
@@ -88,7 +353,6 @@ func main() {
 			fmt.Printf("Error when trying to decode: %v\n", err)
 			fmt.Printf("Verify if the JSON File has UTF8 encoding")
 		}
-
 		if log.Action == "pass" || log.Action == "fail" {
 			if log.Test == "" {
 				finalStatus = log
@@ -98,7 +362,6 @@ func main() {
 					hasFailures = true
 				}
 				test, exists := FindTest(testList, log.Test)
-
 				if exists {
 					if log.Action == "fail" {
 						test.TestLog = log
@@ -117,7 +380,6 @@ func main() {
 			fmt.Printf("Error when trying to open: %v\n", filepath.Join(filepath.ToSlash(testPath), testName))
 			os.Exit(1)
 		}
-
 		decoder2 := json.NewDecoder(jsonTestsOutputClean)
 		for decoder2.More() {
 			var log TestLog
@@ -132,13 +394,28 @@ func main() {
 			}
 
 			test, exists := FindTest(testList, log.Test)
-
 			if !exists || test.TestLog.Action != "fail" {
 				continue
 			}
-
 			test.FailLog = append(test.FailLog, log.Output)
 		}
+
+		for i := range testList {
+			test := &testList[i]
+			if test.TestLog.Action != "fail" {
+				continue
+			}
+
+			if isExpectedVsActual(test.FailLog) {
+				expectedActual := extractExpectedActualLines(test.FailLog)
+				compareMessageContent(&expectedActual)
+				test.ExpectedActual = expectedActual
+			} else if isDifferentNumberOfLines(test.FailLog) {
+				expectedActual := extractPayloadDiffLines(test.FailLog)
+				compareMessageContent(&expectedActual)
+				test.ExpectedActual = expectedActual
+			}
+		}
 	}
 
 	fmt.Printf("Parsing tests data... Done!\n")