Add missing attributes and expose system outputs (#254)

* fix: add missing attributes to parsed model

* fix: extend test_report models

* feat: expose system-out and system-err and append them to message contents

* test: fix tests

* feat: separate Failure, Error, system-out and system-err

Author: zoltanszabo-bitrise

test/converters/converters_test.go

@@ -26,6 +26,7 @@ func TestXCresult3Converters(t *testing.T) {
 				Name:     "rtgtrghtrgTests",
 				Tests:    2,
 				Failures: 0,
+				Errors:   0,
 				Time:     0.26063,
 				TestCases: []testreport.TestCase{
 					{ // plain test case
@@ -44,6 +45,7 @@ func TestXCresult3Converters(t *testing.T) {
 				Name:     "rtgtrghtrgUITests",
 				Tests:    15,
 				Failures: 3,
+				Errors:   0,
 				Time:     0.759,
 				TestCases: []testreport.TestCase{
 					// class rtgtrghtrg3UITests: XCTestCase inside rtgtrghtrgUITests class

test/converters/junitxml/junitxml.go

@@ -5,7 +5,7 @@ import (
 	"strings"
 
 	"github.com/bitrise-steplib/steps-deploy-to-bitrise-io/test/testreport"
-	"github.com/pkg/errors"
+	errorPkg "github.com/pkg/errors"
 )
 
 func (c *Converter) Setup(_ bool) {}
@@ -54,13 +54,9 @@ func parseTestReport(result resultReader) (TestReport, error) {
 		return TestReport{TestSuites: []TestSuite{testSuite}}, nil
 	}
 
-	return TestReport{}, errors.Wrap(errors.Wrap(testSuiteErr, string(data)), testReportErr.Error())
+	return TestReport{}, errorPkg.Wrap(errorPkg.Wrap(testSuiteErr, string(data)), testReportErr.Error())
 }
 
-// merges Suites->Cases->Error and Suites->Cases->SystemErr field values into Suites->Cases->Failure field
-// with 2 newlines and error category prefix
-// the two newlines applied only if there is a failure message already
-// this is required because our testing service currently handles failure field properly
 func convertTestReport(report TestReport) testreport.TestReport {
 	convertedReport := testreport.TestReport{
 		XMLName: report.XMLName,
@@ -76,13 +72,17 @@ func convertTestReport(report TestReport) testreport.TestReport {
 
 func convertTestSuite(testSuite TestSuite) testreport.TestSuite {
 	convertedTestSuite := testreport.TestSuite{
-		XMLName: testSuite.XMLName,
-		Name:    testSuite.Name,
-		Time:    testSuite.Time,
+		XMLName:    testSuite.XMLName,
+		Name:       testSuite.Name,
+		Time:       testSuite.Time,
+		Assertions: testSuite.Assertions,
+		Timestamp:  testSuite.Timestamp,
+		File:       testSuite.File,
 	}
 
 	tests := 0
 	failures := 0
+	errors := 0
 	skipped := 0
 
 	flattenedTestCases := flattenGroupedTestCases(testSuite.TestCases)
@@ -93,6 +93,9 @@ func convertTestSuite(testSuite TestSuite) testreport.TestSuite {
 		if convertedTestCase.Failure != nil {
 			failures++
 		}
+		if convertedTestCase.Error != nil {
+			errors++
+		}
 		if convertedTestCase.Skipped != nil {
 			skipped++
 		}
@@ -101,12 +104,12 @@ func convertTestSuite(testSuite TestSuite) testreport.TestSuite {
 
 	for _, childSuite := range testSuite.TestSuites {
 		convertedChildSuite := convertTestSuite(childSuite)
-		
 		convertedTestSuite.TestSuites = append(convertedTestSuite.TestSuites, convertedChildSuite)
 	}
 
 	convertedTestSuite.Tests = tests
 	convertedTestSuite.Failures = failures
+	convertedTestSuite.Errors = errors
 	convertedTestSuite.Skipped = skipped
 
 	return convertedTestSuite
@@ -130,74 +133,74 @@ func flattenGroupedTestCases(testCases []TestCase) []TestCase {
 		}
 
 		for _, flakyFailure := range testCase.FlakyFailures {
-			flattenedTestCase.Failure = convertToFailure(flakyFailure.Type, flakyFailure.Message, flakyFailure.SystemErr)
+			flattenedTestCase.Failure = &Failure{
+				Type:    flakyFailure.Type,
+				Message: flakyFailure.Message,
+				Value:   flakyFailure.Value,
+			}
+			flattenedTestCase.SystemErr = flakyFailure.SystemErr
+			flattenedTestCase.SystemOut = flakyFailure.SystemOut
 			flattenedTestCases = append(flattenedTestCases, flattenedTestCase)
 		}
 
+		flattenedTestCase.Failure = nil
 		for _, flakyError := range testCase.FlakyErrors {
-			flattenedTestCase.Failure = convertToFailure(flakyError.Type, flakyError.Message, flakyError.SystemErr)
+			flattenedTestCase.Error = &Error{
+				Type:    flakyError.Type,
+				Message: flakyError.Message,
+				Value:   flakyError.Value,
+			}
+			flattenedTestCase.SystemErr = flakyError.SystemErr
+			flattenedTestCase.SystemOut = flakyError.SystemOut
 			flattenedTestCases = append(flattenedTestCases, flattenedTestCase)
 		}
 
+		flattenedTestCase.Error = nil
 		for _, rerunfailure := range testCase.RerunFailures {
-			flattenedTestCase.Failure = convertToFailure(rerunfailure.Type, rerunfailure.Message, rerunfailure.SystemErr)
+			flattenedTestCase.Failure = &Failure{
+				Type:    rerunfailure.Type,
+				Message: rerunfailure.Message,
+				Value:   rerunfailure.Value,
+			}
+			flattenedTestCase.SystemErr = rerunfailure.SystemErr
+			flattenedTestCase.SystemOut = rerunfailure.SystemOut
 			flattenedTestCases = append(flattenedTestCases, flattenedTestCase)
 		}
 
+		flattenedTestCase.Failure = nil
 		for _, rerunError := range testCase.RerunErrors {
-			flattenedTestCase.Failure = convertToFailure(rerunError.Type, rerunError.Message, rerunError.SystemErr)
+			flattenedTestCase.Error = &Error{
+				Type:    rerunError.Type,
+				Message: rerunError.Message,
+				Value:   rerunError.Value,
+			}
+			flattenedTestCase.SystemErr = rerunError.SystemErr
+			flattenedTestCase.SystemOut = rerunError.SystemOut
 			flattenedTestCases = append(flattenedTestCases, flattenedTestCase)
 		}
 
 	}
 	return flattenedTestCases
 }
 
-func convertToFailure(itemType, failureMessage, systemErr string) *Failure {
-	var message string
-	if len(strings.TrimSpace(itemType)) > 0 {
-		message = itemType
-	}
-	if len(strings.TrimSpace(failureMessage)) > 0 {
-		if len(message) > 0 {
-			message += ": "
-		}
-		message += failureMessage
-	}
-
-	if len(strings.TrimSpace(systemErr)) > 0 {
-		if len(message) > 0 {
-			message += "\n\n"
-		}
-		message += "System error:\n" + systemErr
-	}
-
-	if len(message) > 0 {
-		return &Failure{
-			Value: message,
-		}
-	}
-	return nil
-}
-
 func convertTestCase(testCase TestCase) testreport.TestCase {
 	convertedTestCase := testreport.TestCase{
 		XMLName:           testCase.XMLName,
 		ConfigurationHash: testCase.ConfigurationHash,
 		Name:              testCase.Name,
 		ClassName:         testCase.ClassName,
 		Time:              testCase.Time,
+		Assertions:        testCase.Assertions,
+		File:              testCase.File,
+		Line:              testCase.Line,
+		Failure:           convertFailure(testCase.Failure),
+		Error:             convertError(testCase.Error),
+		Skipped:           convertSkipped(testCase.Skipped),
 		Properties:        convertProperties(testCase.Properties),
+		SystemOut:         convertSystemOut(testCase.SystemOut),
+		SystemErr:         convertSystemErr(testCase.SystemErr),
 	}
 
-	if testCase.Skipped != nil {
-		convertedTestCase.Skipped = &testreport.Skipped{
-			XMLName: testCase.Skipped.XMLName,
-		}
-	}
-
-	convertedTestCase.Failure = convertErrorsToFailure(testCase.Failure, testCase.Error, testCase.SystemErr)
-
 	return convertedTestCase
 }
 
@@ -218,38 +221,62 @@ func convertProperties(properties *Properties) *testreport.Properties {
 	return convertedProperties
 }
 
-func convertErrorsToFailure(failure *Failure, error *Error, systemErr string) *testreport.Failure {
-	var messages []string
+func convertSystemOut(systemOut string) *testreport.SystemOut {
+	if len(strings.TrimSpace(systemOut)) == 0 {
+		return nil
+	}
+
+	return &testreport.SystemOut{
+		XMLName: xml.Name{Local: "system-out"},
+		Value:   strings.TrimSpace(systemOut),
+	}
+}
+
+func convertSystemErr(systemErr string) *testreport.SystemErr {
+	if len(strings.TrimSpace(systemErr)) == 0 {
+		return nil
+	}
 
-	if failure != nil {
-		if len(strings.TrimSpace(failure.Message)) > 0 {
-			messages = append(messages, failure.Message)
-		}
+	return &testreport.SystemErr{
+		XMLName: xml.Name{Local: "system-err"},
+		Value:   strings.TrimSpace(systemErr),
+	}
+}
 
-		if len(strings.TrimSpace(failure.Value)) > 0 {
-			messages = append(messages, failure.Value)
-		}
+func convertSkipped(skipped *Skipped) *testreport.Skipped {
+	if skipped == nil {
+		return nil
 	}
 
-	if error != nil {
-		if len(strings.TrimSpace(error.Message)) > 0 {
-			messages = append(messages, "Error message:\n"+error.Message)
-		}
+	return &testreport.Skipped{
+		XMLName: xml.Name{Local: "skipped"},
+		Message: skipped.Message,
+		Value:   strings.TrimSpace(skipped.Value),
+	}
+}
 
-		if len(strings.TrimSpace(error.Value)) > 0 {
-			messages = append(messages, "Error value:\n"+error.Value)
-		}
+func convertFailure(failure *Failure) *testreport.Failure {
+	if failure == nil {
+		return nil
 	}
 
-	if len(systemErr) > 0 {
-		messages = append(messages, "System error:\n"+systemErr)
+	return &testreport.Failure{
+		XMLName: xml.Name{Local: "failure"},
+		Type:    failure.Type,
+		Message: failure.Message,
+		Value:   strings.TrimSpace(failure.Value),
 	}
+}
 
-	if len(messages) > 0 {
-		return &testreport.Failure{
-			XMLName: xml.Name{Local: "failure"},
-			Value:   strings.Join(messages, "\n\n"),
-		}
+func convertError(error *Error) *testreport.Error {
+	if error == nil {
+		return nil
+	}
+
+	return &testreport.Error{
+		XMLName: xml.Name{Local: "error"},
+		Type:    error.Type,
+		Message: error.Message,
+		Value:   strings.TrimSpace(error.Value),
 	}
-	return nil
 }