✨ Add new output format JUnit (#254)

* 🚧 Introduce new output format JUnit

* deprecated `json` flag option
* added new flag option `output`
* started XML marshalling for JUnit format

* 🐛 Add missing type in OutputValue String method

* 🐛 Print Junit report

* ✨ Generate report in Junit format

* ✅ Test FinalResults for JSON and JUnit

* Apply suggestions from code review

Co-authored-by: Nick Kubala <[email protected]>

* 🔥 Remove unused package errors

* 📝 Add missing parenthesis

* 📝 Add junit to output formats in test help

* 📝 Add output formats section with samples

Co-authored-by: Nick Kubala <[email protected]>

Author: charlyx

README.md

@@ -387,12 +387,83 @@ container_test(
   -f, --force                force run of host driver (without user prompt)
   -h, --help                 help for test
   -i, --image string         path to test image
-  -j, --json                 output test results in json format
       --metadata string      path to image metadata file
+      --no-color             no color in the output
+  -o, --output string        output format for the test report (available format: text, json, junit) (default "text")
       --pull                 force a pull of the image before running tests
   -q, --quiet                flag to suppress output
+      --runtime string       runtime to use with docker driver
       --save                 preserve created containers after test run
-      --test-report string   generate JSON test report and write it to specified file.
-      
+      --test-report string   generate test report and write it to specified file (supported format: json, junit; default: json)
  ```   
 See this [example repo](https://github.com/nkubala/structure-test-examples) for a full working example.
+
+## Output formats
+
+Reports are generated using one of the following output formats: `text`, `json` or `junit`.  
+Formats like `json` and `junit` can also be used to write a report to a specified file using the `--test-report`.
+
+### Output samples
+
+#### Text
+
+```text
+====================================
+====== Test file: config.yaml ======
+====================================
+=== RUN: File Existence Test: whoami
+--- PASS
+duration: 0s
+=== RUN: Metadata Test
+--- PASS
+duration: 0s
+
+=====================================
+============== RESULTS ==============
+=====================================
+Passes:      2
+Failures:    0
+Duration:    0s
+Total tests: 2
+
+PASS
+```
+
+#### JSON
+
+The following sample has been formatted.
+
+```json
+{
+  "Pass": 2,
+  "Fail": 0,
+  "Total": 2,
+  "Duration": 0,
+  "Results": [
+    {
+      "Name": "File Existence Test: whoami",
+      "Pass": true,
+      "Duration": 0
+    },
+    {
+      "Name": "Metadata Test",
+      "Pass": true,
+      "Duration": 0
+    }
+  ]
+}
+```
+
+### JUnit
+
+The following sample has been formatted.
+
+```xml
+<?xml version="1.0"?>
+<testsuites failures="0" tests="2" time="0">
+  <testsuite>
+    <testcase name="File Existence Test: whoami" time="0"/>
+    <testcase name="Metadata Test" time="0"/>
+  </testsuite>
+</testsuites>
+```

cmd/container-structure-test/app/cmd/test.go

@@ -64,7 +64,12 @@ func NewCmdTest(out io.Writer) *cobra.Command {
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			if opts.TestReport != "" {
 				// Force JsonOutput
-				opts.JSON = true
+				if opts.Output == unversioned.Text {
+					opts.JSON = true
+					opts.Output = unversioned.Json
+
+					logrus.Warn("raw text format unsupported for writing output file, defaulting to JSON")
+				}
 				testReportFile, err := os.Create(opts.TestReport)
 				if err != nil {
 					return err
@@ -79,6 +84,10 @@ func NewCmdTest(out io.Writer) *cobra.Command {
 
 			color.NoColor = opts.NoColor
 
+			if opts.JSON {
+				opts.Output = unversioned.Json
+			}
+
 			return run(out)
 		},
 	}
@@ -132,12 +141,12 @@ func run(out io.Writer) error {
 	channel := make(chan interface{}, 1)
 	go runTests(out, channel, args, driverImpl)
 	// TODO(nkubala): put a sync.WaitGroup here
-	return test.ProcessResults(out, opts.JSON, channel)
+	return test.ProcessResults(out, opts.Output, channel)
 }
 
 func runTests(out io.Writer, channel chan interface{}, args *drivers.DriverConfig, driverImpl func(drivers.DriverConfig) (drivers.Driver, error)) {
 	for _, file := range opts.ConfigFiles {
-		if !opts.JSON {
+		if opts.Output == unversioned.Text {
 			output.Banner(out, file)
 		}
 		tests, err := test.Parse(file, args, driverImpl)
@@ -190,9 +199,11 @@ func AddTestFlags(cmd *cobra.Command) {
 	cmd.Flags().BoolVarP(&opts.Quiet, "quiet", "q", false, "flag to suppress output")
 	cmd.Flags().BoolVarP(&opts.Force, "force", "f", false, "force run of host driver (without user prompt)")
 	cmd.Flags().BoolVarP(&opts.JSON, "json", "j", false, "output test results in json format")
+	cmd.Flags().MarkDeprecated("json", "please use --output instead")
+	cmd.Flags().VarP(&opts.Output, "output", "o", "output format for the test report (available format: text, json, junit)")
 	cmd.Flags().BoolVar(&opts.NoColor, "no-color", false, "no color in the output")
 
 	cmd.Flags().StringArrayVarP(&opts.ConfigFiles, "config", "c", []string{}, "test config files")
 	cmd.MarkFlagRequired("config")
-	cmd.Flags().StringVar(&opts.TestReport, "test-report", "", "generate JSON test report and write it to specified file.")
+	cmd.Flags().StringVar(&opts.TestReport, "test-report", "", "generate test report and write it to specified file (supported format: json, junit; default: json)")
 }

cmd/container-structure-test/app/cmd/test/util.go

@@ -105,7 +105,7 @@ func Parse(fp string, args *drivers.DriverConfig, driverImpl func(drivers.Driver
 	return tests, nil
 }
 
-func ProcessResults(out io.Writer, json bool, c chan interface{}) error {
+func ProcessResults(out io.Writer, format unversioned.OutputValue, c chan interface{}) error {
 	totalPass := 0
 	totalFail := 0
 	totalDuration := time.Duration(0)
@@ -115,7 +115,7 @@ func ProcessResults(out io.Writer, json bool, c chan interface{}) error {
 		return errors.Wrap(err, "reading results from channel")
 	}
 	for _, r := range results {
-		if !json {
+		if format == unversioned.Text {
 			// output individual results if we're not in json mode
 			output.OutputResult(out, r)
 		}
@@ -139,11 +139,11 @@ func ProcessResults(out io.Writer, json bool, c chan interface{}) error {
 		Fail:     totalFail,
 		Duration: totalDuration,
 	}
-	if json {
+	if format == unversioned.Json || format == unversioned.Junit {
 		// only output results here if we're in json mode
 		summary.Results = results
 	}
-	output.FinalResults(out, json, summary)
+	output.FinalResults(out, format, summary)
 
 	return err
 }

pkg/config/options.go

@@ -14,6 +14,8 @@
 
 package config
 
+import "github.com/GoogleContainerTools/container-structure-test/pkg/types/unversioned"
+
 type StructureTestOptions struct {
 	ImagePath   string
 	Driver      string
@@ -23,6 +25,7 @@ type StructureTestOptions struct {
 	ConfigFiles []string
 
 	JSON    bool
+	Output  unversioned.OutputValue
 	Pull    bool
 	Save    bool
 	Quiet   bool

pkg/output/output.go

@@ -16,6 +16,7 @@ package output
 
 import (
 	"encoding/json"
+	"encoding/xml"
 	"fmt"
 	"io"
 	"path/filepath"
@@ -56,8 +57,8 @@ func Banner(out io.Writer, filename string) {
 	color.Purple.Fprintln(out, strings.Repeat("=", bannerLength))
 }
 
-func FinalResults(out io.Writer, jsonOut bool, result types.SummaryObject) error {
-	if jsonOut {
+func FinalResults(out io.Writer, format types.OutputValue, result types.SummaryObject) error {
+	if format == types.Json {
 		res, err := json.Marshal(result)
 		if err != nil {
 			return errors.Wrap(err, "marshalling json")
@@ -66,6 +67,17 @@ func FinalResults(out io.Writer, jsonOut bool, result types.SummaryObject) error
 		_, err = out.Write(res)
 		return err
 	}
+
+	if format == types.Junit {
+		res, err := xml.Marshal(result)
+		if err != nil {
+			return errors.Wrap(err, "marshalling xml")
+		}
+		res = append(res, []byte("\n")...)
+		_, err = out.Write(res)
+		return err
+	}
+
 	if bannerLength%2 == 0 {
 		bannerLength++
 	}

pkg/output/output_test.go

@@ -0,0 +1,69 @@
+package output
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/GoogleContainerTools/container-structure-test/pkg/types/unversioned"
+)
+
+func TestFinalResults(t *testing.T) {
+	t.Parallel()
+
+	result := unversioned.SummaryObject{
+		Pass:     1,
+		Fail:     1,
+		Total:    2,
+		Duration: time.Duration(2),
+		Results: []*unversioned.TestResult{
+			{
+				Name:     "my first test",
+				Pass:     true,
+				Stdout:   "it works!",
+				Stderr:   "",
+				Duration: time.Duration(1),
+			},
+			{
+				Name:     "my fail",
+				Pass:     false,
+				Stdout:   "",
+				Stderr:   "this failed",
+				Errors:   []string{"this failed because of that"},
+				Duration: time.Duration(1),
+			},
+		},
+	}
+
+	var finalResultsTests = []struct {
+		actual   *bytes.Buffer
+		format   unversioned.OutputValue
+		expected string
+	}{
+		{
+			actual:   bytes.NewBuffer([]byte{}),
+			format:   unversioned.Junit,
+			expected: `<testsuites failures="1" tests="2" time="2"><testsuite><testcase name="my first test" time="1"></testcase><testcase name="my fail" time="1"><failure>this failed because of that</failure></testcase></testsuite></testsuites>`,
+		},
+		{
+			actual:   bytes.NewBuffer([]byte{}),
+			format:   unversioned.Json,
+			expected: `{"Pass":1,"Fail":1,"Total":2,"Duration":2,"Results":[{"Name":"my first test","Pass":true,"Stdout":"it works!","Duration":1},{"Name":"my fail","Pass":false,"Stderr":"this failed","Errors":["this failed because of that"],"Duration":1}]}`,
+		},
+	}
+
+	for _, test := range finalResultsTests {
+		test := test
+
+		t.Run(test.format.String(), func(t *testing.T) {
+			t.Parallel()
+
+			FinalResults(test.actual, test.format, result)
+
+			if strings.TrimSpace(test.actual.String()) != test.expected {
+				t.Errorf("expected %s but got %s", test.expected, test.actual)
+			}
+		})
+	}
+}

pkg/types/unversioned/types.go

@@ -15,6 +15,7 @@
 package unversioned
 
 import (
+	"encoding/xml"
 	"fmt"
 	"strings"
 	"time"
@@ -43,12 +44,12 @@ type Config struct {
 }
 
 type TestResult struct {
-	Name     string
-	Pass     bool
-	Stdout   string   `json:",omitempty"`
-	Stderr   string   `json:",omitempty"`
-	Errors   []string `json:",omitempty"`
-	Duration time.Duration
+	Name     string        `xml:"name,attr"`
+	Pass     bool          `xml:"-"`
+	Stdout   string        `json:",omitempty" xml:"-"`
+	Stderr   string        `json:",omitempty" xml:"-"`
+	Errors   []string      `json:",omitempty" xml:"failure"`
+	Duration time.Duration `xml:"time,attr"`
 }
 
 func (t *TestResult) String() string {
@@ -86,9 +87,41 @@ func (t *TestResult) IsPass() bool {
 }
 
 type SummaryObject struct {
-	Pass     int
-	Fail     int
-	Total    int
-	Duration time.Duration
-	Results  []*TestResult `json:",omitempty"`
+	XMLName  xml.Name      `json:"-" xml:"testsuites"`
+	Pass     int           `xml:"-"`
+	Fail     int           `xml:"failures,attr"`
+	Total    int           `xml:"tests,attr"`
+	Duration time.Duration `xml:"time,attr"`
+	Results  []*TestResult `json:",omitempty" xml:"testsuite>testcase"`
+}
+
+type OutputValue int
+
+const (
+	Text OutputValue = iota
+	Json
+	Junit
+)
+
+func (o OutputValue) String() string {
+	return [...]string{"text", "json", "junit"}[o]
+}
+
+func (o OutputValue) Type() string {
+	return "string"
+}
+
+func (o *OutputValue) Set(value string) error {
+	switch value {
+	case "text":
+		*o = Text
+	case "json":
+		*o = Json
+	case "junit":
+		*o = Junit
+	default:
+		return fmt.Errorf("unsupported format %s: please select from `text`, `json`, or `junit`", value)
+	}
+
+	return nil
 }